From 9b151d7947b5a21cd5aa8a87a74de4212d066a45 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:22:31 +0930 Subject: [PATCH 001/121] [1password] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/1password --- packages/1password/changelog.yml | 5 ++ .../data_stream/audit_events/fields/ecs.yml | 44 ----------- .../data_stream/item_usages/fields/ecs.yml | 48 ----------- .../signin_attempts/fields/ecs.yml | 50 ------------ packages/1password/docs/README.md | 79 ------------------- packages/1password/manifest.yml | 4 +- 6 files changed, 7 insertions(+), 223 deletions(-) delete mode 100644 packages/1password/data_stream/audit_events/fields/ecs.yml delete mode 100644 packages/1password/data_stream/item_usages/fields/ecs.yml delete mode 100644 packages/1password/data_stream/signin_attempts/fields/ecs.yml diff --git a/packages/1password/changelog.yml b/packages/1password/changelog.yml index 243cef9384e..376d8c5444d 100644 --- a/packages/1password/changelog.yml +++ b/packages/1password/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.29.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.28.0" changes: - description: Improve handling of empty responses. diff --git a/packages/1password/data_stream/audit_events/fields/ecs.yml b/packages/1password/data_stream/audit_events/fields/ecs.yml deleted file mode 100644 index c8cee87db6b..00000000000 --- a/packages/1password/data_stream/audit_events/fields/ecs.yml +++ /dev/null @@ -1,44 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: related.user -- external: ecs - name: related.ip -- external: ecs - name: event.kind -- external: ecs - name: event.category -- external: ecs - name: event.type -- external: ecs - name: event.created -- external: ecs - name: event.action -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user.email -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: tags diff --git a/packages/1password/data_stream/item_usages/fields/ecs.yml b/packages/1password/data_stream/item_usages/fields/ecs.yml deleted file mode 100644 index 72692451bb2..00000000000 --- a/packages/1password/data_stream/item_usages/fields/ecs.yml +++ /dev/null @@ -1,48 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: related.user -- external: ecs - name: related.ip -- external: ecs - name: event.kind -- external: ecs - name: event.category -- external: ecs - name: event.type -- external: ecs - name: event.created -- external: ecs - name: event.action -- external: ecs - name: user.id -- external: ecs - name: user.full_name -- external: ecs - name: user.email -- external: ecs - name: host.os.name -- external: ecs - name: host.os.version -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: tags diff --git a/packages/1password/data_stream/signin_attempts/fields/ecs.yml b/packages/1password/data_stream/signin_attempts/fields/ecs.yml deleted file mode 100644 index 4f2aa0facb7..00000000000 --- a/packages/1password/data_stream/signin_attempts/fields/ecs.yml +++ /dev/null @@ -1,50 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: related.user -- external: ecs - name: related.ip -- external: ecs - name: event.kind -- external: ecs - name: event.category -- external: ecs - name: event.type -- external: ecs - name: event.action -- external: ecs - name: event.outcome -- external: ecs - name: event.created -- external: ecs - name: user.id -- external: ecs - name: user.full_name -- external: ecs - name: user.email -- external: ecs - name: host.os.name -- external: ecs - name: host.os.version -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: tags diff --git a/packages/1password/docs/README.md b/packages/1password/docs/README.md index 70eb4954f86..70b7836aaa7 100644 --- a/packages/1password/docs/README.md +++ b/packages/1password/docs/README.md @@ -31,18 +31,8 @@ Use the 1Password Events API to retrieve information about sign-in attempts. Eve | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.version | Operating system version as a raw string. | keyword | | input.type | Input type | keyword | | onepassword.client.app_name | The name of the 1Password app that attempted to sign in to the account | keyword | | onepassword.client.app_version | The version number of the 1Password app | keyword | @@ -53,24 +43,6 @@ Use the 1Password Events API to retrieve information about sign-in attempts. Eve | onepassword.session_uuid | The UUID of the session that created the event | keyword | | onepassword.type | Details about the sign-in attempt | keyword | | onepassword.uuid | The UUID of the event | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| user.email | User email address. | keyword | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | An example event for `signin_attempts` looks as following: @@ -174,17 +146,8 @@ This uses the 1Password Events API to retrieve information about items in shared | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.version | Operating system version as a raw string. | keyword | | input.type | Input type | keyword | | onepassword.client.app_name | The name of the 1Password app the item was accessed from | keyword | | onepassword.client.app_version | The version number of the 1Password app | keyword | @@ -194,24 +157,6 @@ This uses the 1Password Events API to retrieve information about items in shared | onepassword.used_version | The version of the item that was accessed | integer | | onepassword.uuid | The UUID of the event | keyword | | onepassword.vault_uuid | The UUID of the vault the item is in | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| user.email | User email address. | keyword | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | An example event for `item_usages` looks as following: @@ -315,14 +260,8 @@ This uses the 1Password Events API to retrieve information about audit events. E | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | input.type | Input type | keyword | | onepassword.actor_details.email | The email of the actor. | keyword | | onepassword.actor_details.name | The name of the actor. | keyword | @@ -343,24 +282,6 @@ This uses the 1Password Events API to retrieve information about audit events. E | onepassword.session.login_time | The login time of the session used to create the event. | date | | onepassword.session.uuid | The session uuid of the session used to create the event. | keyword | | onepassword.uuid | The UUID of the event. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | An example event for `audit_events` looks as following: diff --git a/packages/1password/manifest.yml b/packages/1password/manifest.yml index d5727968f2f..252412719fa 100644 --- a/packages/1password/manifest.yml +++ b/packages/1password/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: 1password title: "1Password" -version: "1.28.0" +version: "1.29.0" description: Collect logs from 1Password with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - credential_management conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" screenshots: - src: /img/1password-signinattempts-screenshot.png title: Sign-in attempts From 447cd639d0a009ca6b7f2c1b120829156641011e Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:22:32 +0930 Subject: [PATCH 002/121] [akamai] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/akamai --- packages/akamai/changelog.yml | 5 + .../akamai/data_stream/siem/fields/agent.yml | 93 +------------ .../akamai/data_stream/siem/fields/beats.yml | 3 - .../akamai/data_stream/siem/fields/ecs.yml | 126 ------------------ packages/akamai/docs/README.md | 81 ----------- packages/akamai/manifest.yml | 4 +- 6 files changed, 8 insertions(+), 304 deletions(-) delete mode 100644 packages/akamai/data_stream/siem/fields/ecs.yml diff --git a/packages/akamai/changelog.yml b/packages/akamai/changelog.yml index 639790bd9f0..abd2a2b028a 100644 --- a/packages/akamai/changelog.yml +++ b/packages/akamai/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.24.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.23.2" changes: - description: Handle HTTP headers without values. diff --git a/packages/akamai/data_stream/siem/fields/agent.yml b/packages/akamai/data_stream/siem/fields/agent.yml index 4d9a6f7b362..bc42d0a853b 100644 --- a/packages/akamai/data_stream/siem/fields/agent.yml +++ b/packages/akamai/data_stream/siem/fields/agent.yml @@ -1,100 +1,9 @@ - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/akamai/data_stream/siem/fields/beats.yml b/packages/akamai/data_stream/siem/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/akamai/data_stream/siem/fields/beats.yml +++ b/packages/akamai/data_stream/siem/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/akamai/data_stream/siem/fields/ecs.yml b/packages/akamai/data_stream/siem/fields/ecs.yml deleted file mode 100644 index dafaa93238b..00000000000 --- a/packages/akamai/data_stream/siem/fields/ecs.yml +++ /dev/null @@ -1,126 +0,0 @@ -- name: client.as.number - external: ecs -- name: client.as.organization.name - external: ecs -- name: client.domain - external: ecs -- name: client.geo.city_name - external: ecs -- name: client.geo.country_name - external: ecs -- name: client.geo.country_iso_code - external: ecs -- name: client.geo.continent_name - external: ecs -- name: client.geo.region_iso_code - external: ecs -- name: client.geo.location - external: ecs -- name: client.geo.region_name - external: ecs -- name: client.ip - external: ecs -- name: client.address - external: ecs -- name: client.bytes - external: ecs -- name: client.port - external: ecs -- name: ecs.version - external: ecs -- name: error.message - external: ecs -- name: event.action - external: ecs -- name: event.category - external: ecs -- name: event.ingested - external: ecs -- name: event.created - external: ecs -- name: event.start - external: ecs -- name: event.kind - external: ecs -- name: event.original - external: ecs -- name: event.outcome - external: ecs -- name: event.type - external: ecs -- name: related.ip - external: ecs -- name: source.address - external: ecs -- name: source.as.number - external: ecs -- name: source.as.organization.name - external: ecs -- name: source.bytes - external: ecs -- name: source.domain - external: ecs -- name: source.geo.city_name - external: ecs -- name: source.geo.continent_name - external: ecs -- name: source.geo.country_iso_code - external: ecs -- name: source.geo.country_name - external: ecs -- name: source.geo.location - external: ecs -- name: source.geo.name - external: ecs -- name: source.geo.region_iso_code - external: ecs -- name: source.geo.region_name - external: ecs -- name: source.ip - external: ecs -- name: source.port - external: ecs -- name: tags - external: ecs -- name: url.domain - external: ecs -- name: url.password - external: ecs -- name: url.port - external: ecs -- name: url.username - external: ecs -- name: url.path - external: ecs -- name: url.query - external: ecs -- name: url.extension - external: ecs -- name: url.scheme - external: ecs -- name: url.full - external: ecs -- name: tls.cipher - external: ecs -- name: tls.version - external: ecs -- name: tls.version_protocol - external: ecs -- name: network.protocol - external: ecs -- name: network.transport - external: ecs -- name: http.response.status_code - external: ecs -- name: http.response.bytes - external: ecs -- name: http.request.method - external: ecs -- name: http.request.id - external: ecs -- name: http.version - external: ecs -- name: observer.type - external: ecs -- name: observer.vendor - external: ecs diff --git a/packages/akamai/docs/README.md b/packages/akamai/docs/README.md index 43074ae0ed7..dfcb3bd3c73 100644 --- a/packages/akamai/docs/README.md +++ b/packages/akamai/docs/README.md @@ -57,98 +57,17 @@ See [Akamai API get started](https://techdocs.akamai.com/siem-integration/refere | akamai.siem.user_risk.status | Status code indicating any errors that might have occurred when calculating the risk score. | long | | akamai.siem.user_risk.trust | Indicators that were trusted. For example, the value ugp indicates that the user’s country or area is trusted. | flattened | | akamai.siem.user_risk.uuid | Unique identifier of the user whose risk data is being provided. | keyword | -| client.address | Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| client.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| client.as.organization.name | Organization name. | keyword | -| client.as.organization.name.text | Multi-field of `client.as.organization.name`. | match_only_text | -| client.bytes | Bytes sent from the client to the server. | long | -| client.domain | The domain name of the client system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| client.geo.city_name | City name. | keyword | -| client.geo.continent_name | Name of the continent. | keyword | -| client.geo.country_iso_code | Country ISO code. | keyword | -| client.geo.country_name | Country name. | keyword | -| client.geo.location | Longitude and latitude. | geo_point | -| client.geo.region_iso_code | Region ISO code. | keyword | -| client.geo.region_name | Region name. | keyword | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| client.port | Port of the client. | long | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.id | A unique identifier for each HTTP request to correlate logs between clients and servers in transactions. The id may be contained in a non-standard HTTP header, such as `X-Request-ID` or `X-Correlation-ID`. | keyword | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.response.bytes | Total size in bytes of the response (body and headers). | long | -| http.response.status_code | HTTP response status code. | long | -| http.version | HTTP version. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| tls.cipher | String indicating the cipher used during the current connection. | keyword | -| tls.version | Numeric part of the version parsed from the original string. | keyword | -| tls.version_protocol | Normalized lowercase protocol name parsed from original string. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| url.full.text | Multi-field of `url.full`. | match_only_text | -| url.password | Password of the request. | keyword | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.username | Username of the request. | keyword | An example event for `siem` looks as following: diff --git a/packages/akamai/manifest.yml b/packages/akamai/manifest.yml index 7e332f7b83b..370e396285f 100644 --- a/packages/akamai/manifest.yml +++ b/packages/akamai/manifest.yml @@ -1,13 +1,13 @@ name: akamai title: Akamai -version: "2.23.2" +version: "2.24.0" description: Collect logs from Akamai with Elastic Agent. type: integration format_version: "3.0.2" categories: [security, cdn_security] conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" icons: - src: /img/akamai_logo.svg title: Akamai From 1c018cf3af3ce7450fb5f0424e8a754079a5eb80 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:22:49 +0930 Subject: [PATCH 003/121] [amazon_security_lake] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/amazon_security_lake --- packages/amazon_security_lake/_dev/build/build.yml | 1 - packages/amazon_security_lake/changelog.yml | 5 +++++ .../data_stream/application_activity/fields/beats.yml | 6 ------ .../data_stream/discovery/fields/beats.yml | 6 ------ .../amazon_security_lake/data_stream/event/fields/beats.yml | 3 --- .../data_stream/findings/fields/beats.yml | 6 ------ .../amazon_security_lake/data_stream/iam/fields/beats.yml | 6 ------ .../data_stream/network_activity/fields/beats.yml | 6 ------ .../data_stream/system_activity/fields/beats.yml | 6 ------ packages/amazon_security_lake/docs/README.md | 1 - packages/amazon_security_lake/manifest.yml | 4 ++-- 11 files changed, 7 insertions(+), 43 deletions(-) diff --git a/packages/amazon_security_lake/_dev/build/build.yml b/packages/amazon_security_lake/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/amazon_security_lake/_dev/build/build.yml +++ b/packages/amazon_security_lake/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/amazon_security_lake/changelog.yml b/packages/amazon_security_lake/changelog.yml index 882ad77c701..dc0c175a2dd 100644 --- a/packages/amazon_security_lake/changelog.yml +++ b/packages/amazon_security_lake/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.3.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.2.1" changes: - description: Removed SQS support since we don't support sqs based parquet decoding at the input level. diff --git a/packages/amazon_security_lake/data_stream/application_activity/fields/beats.yml b/packages/amazon_security_lake/data_stream/application_activity/fields/beats.yml index 2d2699c8fe1..e2a02e078e8 100644 --- a/packages/amazon_security_lake/data_stream/application_activity/fields/beats.yml +++ b/packages/amazon_security_lake/data_stream/application_activity/fields/beats.yml @@ -7,12 +7,6 @@ - description: Offset of the entry in the log file. name: log.offset type: long -- description: Path to the log file. - name: log.file.path - type: keyword - description: Log message optimized for viewing in a log viewer. name: event.message type: text -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/amazon_security_lake/data_stream/discovery/fields/beats.yml b/packages/amazon_security_lake/data_stream/discovery/fields/beats.yml index 2d2699c8fe1..e2a02e078e8 100644 --- a/packages/amazon_security_lake/data_stream/discovery/fields/beats.yml +++ b/packages/amazon_security_lake/data_stream/discovery/fields/beats.yml @@ -7,12 +7,6 @@ - description: Offset of the entry in the log file. name: log.offset type: long -- description: Path to the log file. - name: log.file.path - type: keyword - description: Log message optimized for viewing in a log viewer. name: event.message type: text -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/amazon_security_lake/data_stream/event/fields/beats.yml b/packages/amazon_security_lake/data_stream/event/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/amazon_security_lake/data_stream/event/fields/beats.yml +++ b/packages/amazon_security_lake/data_stream/event/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/amazon_security_lake/data_stream/findings/fields/beats.yml b/packages/amazon_security_lake/data_stream/findings/fields/beats.yml index 2d2699c8fe1..e2a02e078e8 100644 --- a/packages/amazon_security_lake/data_stream/findings/fields/beats.yml +++ b/packages/amazon_security_lake/data_stream/findings/fields/beats.yml @@ -7,12 +7,6 @@ - description: Offset of the entry in the log file. name: log.offset type: long -- description: Path to the log file. - name: log.file.path - type: keyword - description: Log message optimized for viewing in a log viewer. name: event.message type: text -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/amazon_security_lake/data_stream/iam/fields/beats.yml b/packages/amazon_security_lake/data_stream/iam/fields/beats.yml index 2d2699c8fe1..e2a02e078e8 100644 --- a/packages/amazon_security_lake/data_stream/iam/fields/beats.yml +++ b/packages/amazon_security_lake/data_stream/iam/fields/beats.yml @@ -7,12 +7,6 @@ - description: Offset of the entry in the log file. name: log.offset type: long -- description: Path to the log file. - name: log.file.path - type: keyword - description: Log message optimized for viewing in a log viewer. name: event.message type: text -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/amazon_security_lake/data_stream/network_activity/fields/beats.yml b/packages/amazon_security_lake/data_stream/network_activity/fields/beats.yml index 2d2699c8fe1..e2a02e078e8 100644 --- a/packages/amazon_security_lake/data_stream/network_activity/fields/beats.yml +++ b/packages/amazon_security_lake/data_stream/network_activity/fields/beats.yml @@ -7,12 +7,6 @@ - description: Offset of the entry in the log file. name: log.offset type: long -- description: Path to the log file. - name: log.file.path - type: keyword - description: Log message optimized for viewing in a log viewer. name: event.message type: text -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/amazon_security_lake/data_stream/system_activity/fields/beats.yml b/packages/amazon_security_lake/data_stream/system_activity/fields/beats.yml index 2d2699c8fe1..e2a02e078e8 100644 --- a/packages/amazon_security_lake/data_stream/system_activity/fields/beats.yml +++ b/packages/amazon_security_lake/data_stream/system_activity/fields/beats.yml @@ -7,12 +7,6 @@ - description: Offset of the entry in the log file. name: log.offset type: long -- description: Path to the log file. - name: log.file.path - type: keyword - description: Log message optimized for viewing in a log viewer. name: event.message type: text -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/amazon_security_lake/docs/README.md b/packages/amazon_security_lake/docs/README.md index a89851deb77..11926e7b611 100644 --- a/packages/amazon_security_lake/docs/README.md +++ b/packages/amazon_security_lake/docs/README.md @@ -1960,4 +1960,3 @@ This is the `Event` dataset. | process.user.full_name | | keyword | | process.user.group.id | | keyword | | process.user.group.name | | keyword | -| tags | User defined tags. | keyword | diff --git a/packages/amazon_security_lake/manifest.yml b/packages/amazon_security_lake/manifest.yml index 59714638667..e5c91e5a5e2 100644 --- a/packages/amazon_security_lake/manifest.yml +++ b/packages/amazon_security_lake/manifest.yml @@ -1,13 +1,13 @@ format_version: "3.0.3" name: amazon_security_lake title: Amazon Security Lake -version: "1.2.1" +version: "1.3.0" description: Collect logs from Amazon Security Lake with Elastic Agent. type: integration categories: ["aws", "security"] conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: subscription: basic screenshots: From 731f66e39fec4485a4e4627bdc18d4c86a0da653 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:22:52 +0930 Subject: [PATCH 004/121] [atlassian_bitbucket] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/atlassian_bitbucket --- packages/atlassian_bitbucket/changelog.yml | 5 + .../data_stream/audit/fields/agent.yml | 167 +----------------- .../data_stream/audit/fields/ecs.yml | 70 -------- packages/atlassian_bitbucket/docs/README.md | 68 ------- packages/atlassian_bitbucket/manifest.yml | 4 +- 5 files changed, 8 insertions(+), 306 deletions(-) delete mode 100644 packages/atlassian_bitbucket/data_stream/audit/fields/ecs.yml diff --git a/packages/atlassian_bitbucket/changelog.yml b/packages/atlassian_bitbucket/changelog.yml index 2a6caed7174..82b7365c2af 100644 --- a/packages/atlassian_bitbucket/changelog.yml +++ b/packages/atlassian_bitbucket/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.1.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.0.0" changes: - description: Make `event.type` field conform to ECS field definition. diff --git a/packages/atlassian_bitbucket/data_stream/audit/fields/agent.yml b/packages/atlassian_bitbucket/data_stream/audit/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/atlassian_bitbucket/data_stream/audit/fields/agent.yml +++ b/packages/atlassian_bitbucket/data_stream/audit/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/atlassian_bitbucket/data_stream/audit/fields/ecs.yml b/packages/atlassian_bitbucket/data_stream/audit/fields/ecs.yml deleted file mode 100644 index 97d95f430af..00000000000 --- a/packages/atlassian_bitbucket/data_stream/audit/fields/ecs.yml +++ /dev/null @@ -1,70 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.created -- external: ecs - name: tags -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user.full_name -- external: ecs - name: user.target.full_name -- external: ecs - name: user.target.name -- external: ecs - name: user.target.group.name -- external: ecs - name: user.target.group.id -- external: ecs - name: user.target.id -- external: ecs - name: user.changes.name -- external: ecs - name: user.changes.full_name -- external: ecs - name: group.name -- external: ecs - name: group.id -- name: source.address - external: ecs -- name: source.as.number - external: ecs -- name: source.as.organization.name - external: ecs -- name: source.bytes - external: ecs -- name: source.domain - external: ecs -- name: source.geo.city_name - external: ecs -- name: source.geo.continent_name - external: ecs -- name: source.geo.country_iso_code - external: ecs -- name: source.geo.country_name - external: ecs -- name: source.geo.location - external: ecs -- name: source.geo.name - external: ecs -- name: source.geo.region_iso_code - external: ecs -- name: source.geo.region_name - external: ecs -- name: source.ip - external: ecs -- name: log.file.path - external: ecs -- name: service.address - external: ecs -- name: related.ip - external: ecs -- name: related.user - external: ecs -- name: related.hosts - external: ecs diff --git a/packages/atlassian_bitbucket/docs/README.md b/packages/atlassian_bitbucket/docs/README.md index e036ed6aa85..de88df20eb9 100644 --- a/packages/atlassian_bitbucket/docs/README.md +++ b/packages/atlassian_bitbucket/docs/README.md @@ -25,85 +25,17 @@ The Bitbucket integration collects audit logs from the audit log files or the au | bitbucket.audit.type.category | Category | keyword | | bitbucket.audit.type.categoryI18nKey | categoryI18nKey | keyword | | bitbucket.audit.type.level | Audit Level | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.module | Event module | constant_keyword | -| group.id | Unique identifier for the group on the system/platform. | keyword | -| group.name | Name of the group. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | Log offset | long | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| user.changes.full_name | User's full name, if available. | keyword | -| user.changes.full_name.text | Multi-field of `user.changes.full_name`. | match_only_text | -| user.changes.name | Short name or login of the user. | keyword | -| user.changes.name.text | Multi-field of `user.changes.name`. | match_only_text | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user.target.full_name | User's full name, if available. | keyword | -| user.target.full_name.text | Multi-field of `user.target.full_name`. | match_only_text | -| user.target.group.id | Unique identifier for the group on the system/platform. | keyword | -| user.target.group.name | Name of the group. | keyword | -| user.target.id | Unique identifier of the user. | keyword | -| user.target.name | Short name or login of the user. | keyword | -| user.target.name.text | Multi-field of `user.target.name`. | match_only_text | An example event for `audit` looks as following: diff --git a/packages/atlassian_bitbucket/manifest.yml b/packages/atlassian_bitbucket/manifest.yml index f181d4e8baf..77dccc738aa 100644 --- a/packages/atlassian_bitbucket/manifest.yml +++ b/packages/atlassian_bitbucket/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: atlassian_bitbucket title: Atlassian Bitbucket -version: "2.0.0" +version: "2.1.0" description: Collect logs from Atlassian Bitbucket with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - productivity_security conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" icons: - src: /img/bitbucket-logo.svg title: Bitbucket Logo From 2fe7ad44a20cced691a692eeb1ed512a094c910f Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:22:58 +0930 Subject: [PATCH 005/121] [atlassian_confluence] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/atlassian_confluence --- packages/atlassian_confluence/changelog.yml | 5 + .../data_stream/audit/fields/agent.yml | 167 +----------------- .../data_stream/audit/fields/ecs.yml | 84 --------- packages/atlassian_confluence/docs/README.md | 75 -------- packages/atlassian_confluence/manifest.yml | 4 +- 5 files changed, 8 insertions(+), 327 deletions(-) delete mode 100644 packages/atlassian_confluence/data_stream/audit/fields/ecs.yml diff --git a/packages/atlassian_confluence/changelog.yml b/packages/atlassian_confluence/changelog.yml index 6c2f78ca1d3..f4cd9a92a9b 100644 --- a/packages/atlassian_confluence/changelog.yml +++ b/packages/atlassian_confluence/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.25.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.24.0" changes: - description: Set sensitive values as secret. diff --git a/packages/atlassian_confluence/data_stream/audit/fields/agent.yml b/packages/atlassian_confluence/data_stream/audit/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/atlassian_confluence/data_stream/audit/fields/agent.yml +++ b/packages/atlassian_confluence/data_stream/audit/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/atlassian_confluence/data_stream/audit/fields/ecs.yml b/packages/atlassian_confluence/data_stream/audit/fields/ecs.yml deleted file mode 100644 index 49469eb61eb..00000000000 --- a/packages/atlassian_confluence/data_stream/audit/fields/ecs.yml +++ /dev/null @@ -1,84 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: group.id -- external: ecs - name: group.name -- external: ecs - name: log.file.path -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: service.address -- external: ecs - name: source.address -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.bytes -- external: ecs - name: source.domain -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.name -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: tags -- external: ecs - name: user.changes.email -- external: ecs - name: user.changes.full_name -- external: ecs - name: user.changes.name -- external: ecs - name: user.full_name -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user.target.email -- external: ecs - name: user.target.full_name -- external: ecs - name: user.target.group.id -- external: ecs - name: user.target.group.name -- external: ecs - name: user.target.id -- external: ecs - name: user.target.name diff --git a/packages/atlassian_confluence/docs/README.md b/packages/atlassian_confluence/docs/README.md index 06e7f9e9bce..cbc3e6425f0 100644 --- a/packages/atlassian_confluence/docs/README.md +++ b/packages/atlassian_confluence/docs/README.md @@ -19,15 +19,7 @@ The Confluence integration collects audit logs from the audit log files or the a | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | | confluence.audit.affected_objects | Affected Objects | flattened | | confluence.audit.changed_values | Changed Values | flattened | | confluence.audit.external_collaborator | Whether the user is an external collaborator user | boolean | @@ -39,83 +31,16 @@ The Confluence integration collects audit logs from the audit log files or the a | confluence.audit.type.category | Category | keyword | | confluence.audit.type.categoryI18nKey | categoryI18nKey | keyword | | confluence.audit.type.level | Audit Level | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| group.id | Unique identifier for the group on the system/platform. | keyword | -| group.name | Name of the group. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | Log offset | long | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| user.changes.email | User email address. | keyword | -| user.changes.full_name | User's full name, if available. | keyword | -| user.changes.full_name.text | Multi-field of `user.changes.full_name`. | match_only_text | -| user.changes.name | Short name or login of the user. | keyword | -| user.changes.name.text | Multi-field of `user.changes.name`. | match_only_text | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user.target.email | User email address. | keyword | -| user.target.full_name | User's full name, if available. | keyword | -| user.target.full_name.text | Multi-field of `user.target.full_name`. | match_only_text | -| user.target.group.id | Unique identifier for the group on the system/platform. | keyword | -| user.target.group.name | Name of the group. | keyword | -| user.target.id | Unique identifier of the user. | keyword | -| user.target.name | Short name or login of the user. | keyword | -| user.target.name.text | Multi-field of `user.target.name`. | match_only_text | An example event for `audit` looks as following: diff --git a/packages/atlassian_confluence/manifest.yml b/packages/atlassian_confluence/manifest.yml index 2a69a95862d..1db4b34d02c 100644 --- a/packages/atlassian_confluence/manifest.yml +++ b/packages/atlassian_confluence/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: atlassian_confluence title: Atlassian Confluence -version: "1.24.0" +version: "1.25.0" description: Collect logs from Atlassian Confluence with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - productivity_security conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" icons: - src: /img/confluence-logo.svg title: Confluence Logo From 7bc15c59a524707b77436bab0f54287f054a882a Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:01 +0930 Subject: [PATCH 006/121] [atlassian_jira] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/atlassian_jira --- packages/atlassian_jira/changelog.yml | 5 + .../data_stream/audit/fields/agent.yml | 167 +----------------- .../data_stream/audit/fields/ecs.yml | 84 --------- packages/atlassian_jira/docs/README.md | 75 -------- packages/atlassian_jira/manifest.yml | 4 +- 5 files changed, 8 insertions(+), 327 deletions(-) delete mode 100644 packages/atlassian_jira/data_stream/audit/fields/ecs.yml diff --git a/packages/atlassian_jira/changelog.yml b/packages/atlassian_jira/changelog.yml index 4deb00a2021..e45e96a061b 100644 --- a/packages/atlassian_jira/changelog.yml +++ b/packages/atlassian_jira/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.26.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.25.0" changes: - description: Improve handling of empty responses. diff --git a/packages/atlassian_jira/data_stream/audit/fields/agent.yml b/packages/atlassian_jira/data_stream/audit/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/atlassian_jira/data_stream/audit/fields/agent.yml +++ b/packages/atlassian_jira/data_stream/audit/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/atlassian_jira/data_stream/audit/fields/ecs.yml b/packages/atlassian_jira/data_stream/audit/fields/ecs.yml deleted file mode 100644 index 467fad1fd0b..00000000000 --- a/packages/atlassian_jira/data_stream/audit/fields/ecs.yml +++ /dev/null @@ -1,84 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.type -- external: ecs - name: group.name -- external: ecs - name: log.file.path -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: service.address -- external: ecs - name: source.address -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.bytes -- external: ecs - name: source.domain -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.name -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: tags -- external: ecs - name: user.changes.email -- external: ecs - name: user.changes.full_name -- external: ecs - name: user.changes.name -- external: ecs - name: user.full_name -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user.target.email -- external: ecs - name: user.target.full_name -- external: ecs - name: user.target.group.name -- external: ecs - name: user.target.id -- external: ecs - name: user.target.name diff --git a/packages/atlassian_jira/docs/README.md b/packages/atlassian_jira/docs/README.md index 14c04d88b18..aa3fb18d168 100644 --- a/packages/atlassian_jira/docs/README.md +++ b/packages/atlassian_jira/docs/README.md @@ -19,52 +19,15 @@ The Jira integration collects audit logs from the audit log files or the audit A | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| group.name | Name of the group. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | jira.audit.affected_objects | Affected Objects | flattened | | jira.audit.changed_values | Changed Values | flattened | @@ -76,45 +39,7 @@ The Jira integration collects audit logs from the audit log files or the audit A | jira.audit.type.category | Category | keyword | | jira.audit.type.categoryI18nKey | categoryI18nKey | keyword | | jira.audit.type.level | Audit Level | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | Log offset | long | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| user.changes.email | User email address. | keyword | -| user.changes.full_name | User's full name, if available. | keyword | -| user.changes.full_name.text | Multi-field of `user.changes.full_name`. | match_only_text | -| user.changes.name | Short name or login of the user. | keyword | -| user.changes.name.text | Multi-field of `user.changes.name`. | match_only_text | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user.target.email | User email address. | keyword | -| user.target.full_name | User's full name, if available. | keyword | -| user.target.full_name.text | Multi-field of `user.target.full_name`. | match_only_text | -| user.target.group.name | Name of the group. | keyword | -| user.target.id | Unique identifier of the user. | keyword | -| user.target.name | Short name or login of the user. | keyword | -| user.target.name.text | Multi-field of `user.target.name`. | match_only_text | An example event for `audit` looks as following: diff --git a/packages/atlassian_jira/manifest.yml b/packages/atlassian_jira/manifest.yml index 9f9dc5bc0f1..5c8f9322dca 100644 --- a/packages/atlassian_jira/manifest.yml +++ b/packages/atlassian_jira/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: atlassian_jira title: Atlassian Jira -version: "1.25.0" +version: "1.26.0" description: Collect logs from Atlassian Jira with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - productivity_security conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" icons: - src: /img/jira-software-logo.svg title: Jira Software Logo From 794fc08f463fdfa99fbaafbd1781d8c9df4291ae Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:04 +0930 Subject: [PATCH 007/121] [auth0] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/auth0 --- packages/auth0/changelog.yml | 5 + .../auth0/data_stream/logs/fields/ecs.yml | 128 ------------------ packages/auth0/docs/README.md | 76 ----------- packages/auth0/manifest.yml | 4 +- 4 files changed, 7 insertions(+), 206 deletions(-) delete mode 100644 packages/auth0/data_stream/logs/fields/ecs.yml diff --git a/packages/auth0/changelog.yml b/packages/auth0/changelog.yml index a0a8fa2997c..108a9137866 100644 --- a/packages/auth0/changelog.yml +++ b/packages/auth0/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.16.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.15.0" changes: - description: Set sensitive values as secret. diff --git a/packages/auth0/data_stream/logs/fields/ecs.yml b/packages/auth0/data_stream/logs/fields/ecs.yml deleted file mode 100644 index 0dbcc9a5ffa..00000000000 --- a/packages/auth0/data_stream/logs/fields/ecs.yml +++ /dev/null @@ -1,128 +0,0 @@ -- external: ecs - name: destination.user.domain -- external: ecs - name: destination.user.id -- external: ecs - name: destination.user.name -- external: ecs - name: ecs.version -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.code -- external: ecs - name: event.created -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.outcome -- external: ecs - name: event.original -- external: ecs - name: event.provider -- external: ecs - name: event.sequence -- external: ecs - name: event.type -- external: ecs - name: event.id -- external: ecs - name: file.directory -- external: ecs - name: file.extension -- external: ecs - name: file.name -- external: ecs - name: file.path -- external: ecs - name: host.name -- external: ecs - name: log.level -- external: ecs - name: process.args -- external: ecs - name: process.args_count -- external: ecs - name: process.command_line -- external: ecs - name: process.entity_id -- external: ecs - name: process.executable -- external: ecs - name: process.name -- external: ecs - name: process.pid -- external: ecs - name: process.title -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: source.user.domain -- external: ecs - name: source.user.id -- external: ecs - name: source.user.name -- external: ecs - name: user.domain -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: source.ip -- external: ecs - name: network.type -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.name -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.version -- external: ecs - name: user_agent.os.family -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.kernel -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.platform -- external: ecs - name: user_agent.os.type -- external: ecs - name: user_agent.os.version -- external: ecs - name: tags diff --git a/packages/auth0/docs/README.md b/packages/auth0/docs/README.md index 0d25dae75f8..c6cd7a81b1a 100644 --- a/packages/auth0/docs/README.md +++ b/packages/auth0/docs/README.md @@ -87,85 +87,9 @@ The Auth0 logs dataset provides events from Auth0 log stream. All Auth0 log even | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| destination.user.id | Unique identifier of the user. | keyword | -| destination.user.name | Short name or login of the user. | keyword | -| destination.user.name.text | Multi-field of `destination.user.name`. | match_only_text | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event timestamp. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event timestamp. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.sequence | Sequence number of the event. The sequence number is a value published by some event sources, to make the exact ordering of events unambiguous, regardless of the timestamp precision. | long | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.directory | Directory where the file is located. It should include the drive letter, when appropriate. | keyword | -| file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | input.type | Input type. | keyword | -| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | -| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | -| process.args | Array of process arguments, starting with the absolute path to the executable. May be filtered to protect sensitive information. | keyword | -| process.args_count | Length of the process.args array. This field can be useful for querying or performing bucket analysis on how many arguments were provided to start a process. More arguments may be an indication of suspicious activity. | long | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.executable | Absolute path to the process executable. | keyword | -| process.executable.text | Multi-field of `process.executable`. | match_only_text | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.pid | Process id. | long | -| process.title | Process title. The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. | keyword | -| process.title.text | Multi-field of `process.title`. | match_only_text | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| source.user.id | Unique identifier of the user. | keyword | -| source.user.name | Short name or login of the user. | keyword | -| source.user.name.text | Multi-field of `source.user.name`. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.kernel | Operating system kernel version as a raw string. | keyword | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| user_agent.os.type | Use the `os.type` field to categorize the operating system into one of the broad commercial families. If the OS you're dealing with is not listed as an expected value, the field should not be populated. Please let us know by opening an issue with ECS, to propose its addition. | keyword | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | An example event for `logs` looks as following: diff --git a/packages/auth0/manifest.yml b/packages/auth0/manifest.yml index 3c28fe3fd23..f1108d1a8d2 100644 --- a/packages/auth0/manifest.yml +++ b/packages/auth0/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: auth0 title: "Auth0" -version: "1.15.0" +version: "1.16.0" description: Collect logs from Auth0 with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - iam conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" screenshots: - src: /img/auth0-screenshot.png title: Auth0 Dashboard From a549f106ea15e3e26a7d682aaa156d5ac8469603 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:05 +0930 Subject: [PATCH 008/121] [aws_bedrock] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/aws_bedrock --- packages/aws_bedrock/changelog.yml | 5 ++ .../data_stream/invocation/fields/agent.yml | 53 ------------------- .../data_stream/invocation/fields/ecs.yml | 10 ---- .../data_stream/invocation/fields/input.yml | 2 - packages/aws_bedrock/docs/README.md | 32 ----------- packages/aws_bedrock/manifest.yml | 4 +- 6 files changed, 7 insertions(+), 99 deletions(-) delete mode 100644 packages/aws_bedrock/data_stream/invocation/fields/ecs.yml diff --git a/packages/aws_bedrock/changelog.yml b/packages/aws_bedrock/changelog.yml index dc35ffa9c52..7d90ba0347f 100644 --- a/packages/aws_bedrock/changelog.yml +++ b/packages/aws_bedrock/changelog.yml @@ -1,3 +1,8 @@ +- version: "0.2.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "0.1.3" changes: - description: Fix name canonicalization routines. diff --git a/packages/aws_bedrock/data_stream/invocation/fields/agent.yml b/packages/aws_bedrock/data_stream/invocation/fields/agent.yml index 1efbd1f3b9a..4481cca42e2 100644 --- a/packages/aws_bedrock/data_stream/invocation/fields/agent.yml +++ b/packages/aws_bedrock/data_stream/invocation/fields/agent.yml @@ -1,65 +1,12 @@ - name: cloud type: group fields: - - name: account.id - external: ecs - - name: availability_zone - external: ecs - - name: instance.id - external: ecs - - name: instance.name - external: ecs - - name: machine.type - external: ecs - - name: provider - external: ecs - - name: region - external: ecs - - name: project.id - external: ecs - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - type: group - fields: - - name: id - external: ecs - - name: image.name - external: ecs - - name: labels - external: ecs - - name: name - external: ecs - name: host type: group fields: - - name: architecture - external: ecs - - name: domain - external: ecs - - name: hostname - external: ecs - - name: id - external: ecs - - name: ip - external: ecs - - name: mac - external: ecs - - name: name - external: ecs - - name: os.family - external: ecs - - name: os.kernel - external: ecs - - name: os.name - external: ecs - - name: os.platform - external: ecs - - name: os.version - external: ecs - - name: type - external: ecs - name: containerized type: boolean description: > diff --git a/packages/aws_bedrock/data_stream/invocation/fields/ecs.yml b/packages/aws_bedrock/data_stream/invocation/fields/ecs.yml deleted file mode 100644 index aabaf0c27ab..00000000000 --- a/packages/aws_bedrock/data_stream/invocation/fields/ecs.yml +++ /dev/null @@ -1,10 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- name: event.original - external: ecs -- name: tags - external: ecs -- name: user.id - external: ecs diff --git a/packages/aws_bedrock/data_stream/invocation/fields/input.yml b/packages/aws_bedrock/data_stream/invocation/fields/input.yml index 9710d7290f6..e7adaa1f668 100644 --- a/packages/aws_bedrock/data_stream/invocation/fields/input.yml +++ b/packages/aws_bedrock/data_stream/invocation/fields/input.yml @@ -20,8 +20,6 @@ - name: object.key type: keyword description: Name of the S3 object that this log retrieved from. -- name: log.file.path - external: ecs - name: log.offset type: long description: Log offset diff --git a/packages/aws_bedrock/docs/README.md b/packages/aws_bedrock/docs/README.md index 87f610cc231..97f7abfc060 100644 --- a/packages/aws_bedrock/docs/README.md +++ b/packages/aws_bedrock/docs/README.md @@ -130,26 +130,12 @@ list log events from the specified log group. | aws_bedrock.invocation.schema_type | | keyword | | aws_bedrock.invocation.schema_version | | keyword | | aws_bedrock.invocation.task_type | | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | event.dataset | Event dataset | constant_keyword | | event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | | gen_ai.analysis.action_recommended | Recommended actions based on the analysis. | keyword | | gen_ai.analysis.findings | Detailed findings from security tools. | nested | | gen_ai.analysis.function | Name of the security or analysis function used. | keyword | @@ -214,27 +200,9 @@ list log events from the specified log group. | gen_ai.usage.prompt_tokens | Number of tokens in the user's request. | integer | | gen_ai.user.id | Unique identifier for the user. | keyword | | gen_ai.user.rn | Unique resource name for the user. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| user.id | Unique identifier of the user. | keyword | diff --git a/packages/aws_bedrock/manifest.yml b/packages/aws_bedrock/manifest.yml index 3e1d3f54825..d6a160bfcb9 100644 --- a/packages/aws_bedrock/manifest.yml +++ b/packages/aws_bedrock/manifest.yml @@ -3,12 +3,12 @@ name: aws_bedrock title: AWS Bedrock description: Collect AWS Bedrock model invocation logs with Elastic Agent. type: integration -version: "0.1.3" +version: "0.2.0" categories: - aws conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: subscription: basic policy_templates: From 3a1173ae89569c8d73319213ebfacc612e9e9051 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:05 +0930 Subject: [PATCH 009/121] [azure_blob_storage] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. The ecs.version in sample_event.json files was changed to 8.11.0. Previously sample_event.json files contained 8.0.0. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/azure_blob_storage --- packages/azure_blob_storage/_dev/build/build.yml | 1 - packages/azure_blob_storage/changelog.yml | 5 +++++ packages/azure_blob_storage/fields/agent.yml | 3 --- packages/azure_blob_storage/fields/beats.yml | 3 --- packages/azure_blob_storage/manifest.yml | 7 ++----- packages/azure_blob_storage/sample_event.json | 2 +- 6 files changed, 8 insertions(+), 13 deletions(-) diff --git a/packages/azure_blob_storage/_dev/build/build.yml b/packages/azure_blob_storage/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/azure_blob_storage/_dev/build/build.yml +++ b/packages/azure_blob_storage/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/azure_blob_storage/changelog.yml b/packages/azure_blob_storage/changelog.yml index e0938fa7a04..fd1807dc970 100644 --- a/packages/azure_blob_storage/changelog.yml +++ b/packages/azure_blob_storage/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.1.0" + changes: + - description: ECS version updated to 8.11.0. Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.0.0" changes: - description: Converted Azure Blob Storage to input package type. diff --git a/packages/azure_blob_storage/fields/agent.yml b/packages/azure_blob_storage/fields/agent.yml index 230f7bc911d..9638d2992eb 100644 --- a/packages/azure_blob_storage/fields/agent.yml +++ b/packages/azure_blob_storage/fields/agent.yml @@ -5,9 +5,6 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. diff --git a/packages/azure_blob_storage/fields/beats.yml b/packages/azure_blob_storage/fields/beats.yml index 8c03b061f7c..6d9a7862671 100644 --- a/packages/azure_blob_storage/fields/beats.yml +++ b/packages/azure_blob_storage/fields/beats.yml @@ -1,9 +1,6 @@ - name: input.type description: Type of Filebeat input. type: keyword -- name: tags - type: keyword - description: User defined tags - name: log.offset type: long description: Log offset diff --git a/packages/azure_blob_storage/manifest.yml b/packages/azure_blob_storage/manifest.yml index 56f183ea3db..dfba13faa95 100644 --- a/packages/azure_blob_storage/manifest.yml +++ b/packages/azure_blob_storage/manifest.yml @@ -3,10 +3,10 @@ name: azure_blob_storage title: Custom Azure Blob Storage Input description: Collect log data from configured Azure Blob Storage Container with Elastic Agent. type: input -version: "2.0.0" +version: "2.1.0" conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" categories: - custom - cloud @@ -92,7 +92,6 @@ policy_templates: title: Containers description: > This attribute contains the details about a specific container like, name, number_of_workers, poll, poll_interval etc. The attribute 'name' is specific to a container as it describes the container name, while the fields number_of_workers, poll, poll_interval can exist both at the container level and at the global level. If you have already defined the attributes globally, then you can only specify the container name in this yaml config. If you want to override any specific attribute for a container, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-containers) for further information. - required: true show_user: true default: | @@ -114,7 +113,6 @@ policy_templates: # - regex: "event/" description: > If the container will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the container filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. - - name: timestamp_epoch type: integer title: Timestamp Epoch @@ -129,7 +127,6 @@ policy_templates: show_user: false description: > If the file-set using this input expects to receive multiple messages bundled under a specific field or an array of objects then the config option for 'expand_event_list_from_field' can be specified. This setting will be able to split the messages under the group value into separate events. This can be specified at the global level or at the container level. For more info please refer to the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-azure-blob-storage.html#attrib-expand_event_list_from_field). - - name: preserve_original_event required: true show_user: true diff --git a/packages/azure_blob_storage/sample_event.json b/packages/azure_blob_storage/sample_event.json index f69f9e1774f..29ba5dedc8b 100644 --- a/packages/azure_blob_storage/sample_event.json +++ b/packages/azure_blob_storage/sample_event.json @@ -27,7 +27,7 @@ "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "08985f2a-c29f-4867-90dc-787df2a6e4ce", From 86879b9b402295e13c523a19c70eb7ab95171ba3 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:07 +0930 Subject: [PATCH 010/121] [azure_frontdoor] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/azure_frontdoor --- packages/azure_frontdoor/changelog.yml | 5 + .../data_stream/access/fields/agent.yml | 50 ------ .../data_stream/access/fields/base-fields.yml | 3 - .../data_stream/access/fields/ecs.yml | 114 ------------ .../data_stream/waf/fields/agent.yml | 50 ------ .../data_stream/waf/fields/base-fields.yml | 3 - .../data_stream/waf/fields/ecs.yml | 90 ---------- packages/azure_frontdoor/docs/README.md | 169 ------------------ packages/azure_frontdoor/manifest.yml | 4 +- 9 files changed, 7 insertions(+), 481 deletions(-) delete mode 100644 packages/azure_frontdoor/data_stream/access/fields/ecs.yml delete mode 100644 packages/azure_frontdoor/data_stream/waf/fields/ecs.yml diff --git a/packages/azure_frontdoor/changelog.yml b/packages/azure_frontdoor/changelog.yml index 95c4f7a0305..8ac051e2d31 100644 --- a/packages/azure_frontdoor/changelog.yml +++ b/packages/azure_frontdoor/changelog.yml @@ -1,3 +1,8 @@ +- version: "1.8.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.7.0" changes: - description: Set sensitive values as secret. diff --git a/packages/azure_frontdoor/data_stream/access/fields/agent.yml b/packages/azure_frontdoor/data_stream/access/fields/agent.yml index bca66ea4ae0..4b15225a4d4 100644 --- a/packages/azure_frontdoor/data_stream/access/fields/agent.yml +++ b/packages/azure_frontdoor/data_stream/access/fields/agent.yml @@ -1,56 +1,6 @@ -- name: cloud.account.id - external: ecs -- name: cloud.availability_zone - external: ecs -- name: cloud.instance.id - external: ecs -- name: cloud.instance.name - external: ecs -- name: cloud.machine.type - external: ecs -- name: cloud.provider - external: ecs -- name: cloud.region - external: ecs -- name: cloud.project.id - external: ecs - name: cloud.image.id type: keyword description: Image ID for the cloud instance. -- name: container.id - external: ecs -- name: container.image.name - external: ecs -- name: container.labels - external: ecs -- name: container.name - external: ecs -- name: host.architecture - external: ecs -- name: host.domain - external: ecs -- name: host.hostname - external: ecs -- name: host.id - external: ecs -- name: host.ip - external: ecs -- name: host.mac - external: ecs -- name: host.name - external: ecs -- name: host.os.family - external: ecs -- name: host.os.kernel - external: ecs -- name: host.os.name - external: ecs -- name: host.os.platform - external: ecs -- name: host.os.version - external: ecs -- name: host.type - external: ecs - name: host.containerized type: boolean description: If the host is a container. diff --git a/packages/azure_frontdoor/data_stream/access/fields/base-fields.yml b/packages/azure_frontdoor/data_stream/access/fields/base-fields.yml index f245714ba9e..208aa91eeb8 100644 --- a/packages/azure_frontdoor/data_stream/access/fields/base-fields.yml +++ b/packages/azure_frontdoor/data_stream/access/fields/base-fields.yml @@ -16,6 +16,3 @@ - name: log.offset type: long description: Log offset. -- name: log.file.path - type: keyword - description: Log file path. diff --git a/packages/azure_frontdoor/data_stream/access/fields/ecs.yml b/packages/azure_frontdoor/data_stream/access/fields/ecs.yml deleted file mode 100644 index f2d91664dec..00000000000 --- a/packages/azure_frontdoor/data_stream/access/fields/ecs.yml +++ /dev/null @@ -1,114 +0,0 @@ -- name: client.ip - external: ecs -- name: client.address - external: ecs -- name: client.port - external: ecs -- name: destination.address - external: ecs -- name: destination.as.number - external: ecs -- name: destination.as.organization.name - external: ecs -- name: destination.ip - external: ecs -- name: destination.port - external: ecs -- name: ecs.version - external: ecs -- name: message - external: ecs -- name: event.action - external: ecs -- name: event.category - external: ecs -- name: event.created - external: ecs -- name: event.id - external: ecs -- name: event.ingested - external: ecs -- name: event.kind - external: ecs -- name: event.type - external: ecs -- name: file.mime_type - external: ecs -- name: file.size - external: ecs -- name: network.community_id - external: ecs -- name: network.protocol - external: ecs -- name: related.ip - external: ecs -- name: related.user - external: ecs -- name: source.address - external: ecs -- name: source.as.number - external: ecs -- name: source.as.organization.name - external: ecs -- name: log.level - external: ecs -- name: source.geo.city_name - external: ecs -- name: source.geo.continent_name - external: ecs -- name: source.geo.country_iso_code - external: ecs -- name: source.geo.country_name - external: ecs -- name: source.geo.location - external: ecs -- name: source.geo.name - external: ecs -- name: source.geo.region_iso_code - external: ecs -- name: source.geo.region_name - external: ecs -- name: source.ip - external: ecs -- name: source.port - external: ecs -- name: user.full_name - external: ecs -- name: user.domain - external: ecs -- name: user.id - external: ecs -- name: user.name - external: ecs -- name: tags - external: ecs -- name: url.original - external: ecs -- name: http.request.bytes - external: ecs -- name: http.request.method - external: ecs -- name: http.response.bytes - external: ecs -- name: http.response.status_code - external: ecs -- name: http.version - external: ecs -- name: tls.version - external: ecs -- name: tls.version_protocol - external: ecs -- name: user_agent.original - external: ecs -- name: user_agent.device.name - external: ecs -- name: user_agent.name - external: ecs -- name: user_agent.os.full - external: ecs -- name: user_agent.os.name - external: ecs -- name: user_agent.os.version - external: ecs -- name: user_agent.version - external: ecs diff --git a/packages/azure_frontdoor/data_stream/waf/fields/agent.yml b/packages/azure_frontdoor/data_stream/waf/fields/agent.yml index bca66ea4ae0..4b15225a4d4 100644 --- a/packages/azure_frontdoor/data_stream/waf/fields/agent.yml +++ b/packages/azure_frontdoor/data_stream/waf/fields/agent.yml @@ -1,56 +1,6 @@ -- name: cloud.account.id - external: ecs -- name: cloud.availability_zone - external: ecs -- name: cloud.instance.id - external: ecs -- name: cloud.instance.name - external: ecs -- name: cloud.machine.type - external: ecs -- name: cloud.provider - external: ecs -- name: cloud.region - external: ecs -- name: cloud.project.id - external: ecs - name: cloud.image.id type: keyword description: Image ID for the cloud instance. -- name: container.id - external: ecs -- name: container.image.name - external: ecs -- name: container.labels - external: ecs -- name: container.name - external: ecs -- name: host.architecture - external: ecs -- name: host.domain - external: ecs -- name: host.hostname - external: ecs -- name: host.id - external: ecs -- name: host.ip - external: ecs -- name: host.mac - external: ecs -- name: host.name - external: ecs -- name: host.os.family - external: ecs -- name: host.os.kernel - external: ecs -- name: host.os.name - external: ecs -- name: host.os.platform - external: ecs -- name: host.os.version - external: ecs -- name: host.type - external: ecs - name: host.containerized type: boolean description: If the host is a container. diff --git a/packages/azure_frontdoor/data_stream/waf/fields/base-fields.yml b/packages/azure_frontdoor/data_stream/waf/fields/base-fields.yml index f245714ba9e..208aa91eeb8 100644 --- a/packages/azure_frontdoor/data_stream/waf/fields/base-fields.yml +++ b/packages/azure_frontdoor/data_stream/waf/fields/base-fields.yml @@ -16,6 +16,3 @@ - name: log.offset type: long description: Log offset. -- name: log.file.path - type: keyword - description: Log file path. diff --git a/packages/azure_frontdoor/data_stream/waf/fields/ecs.yml b/packages/azure_frontdoor/data_stream/waf/fields/ecs.yml deleted file mode 100644 index 7e9667634cb..00000000000 --- a/packages/azure_frontdoor/data_stream/waf/fields/ecs.yml +++ /dev/null @@ -1,90 +0,0 @@ -- name: client.ip - external: ecs -- name: client.address - external: ecs -- name: client.port - external: ecs -- name: destination.address - external: ecs -- name: destination.as.number - external: ecs -- name: destination.as.organization.name - external: ecs -- name: destination.ip - external: ecs -- name: destination.port - external: ecs -- name: ecs.version - external: ecs -- name: message - external: ecs -- name: event.action - external: ecs -- name: event.category - external: ecs -- name: event.created - external: ecs -- name: event.id - external: ecs -- name: event.ingested - external: ecs -- name: event.kind - external: ecs -- name: event.type - external: ecs -- name: file.mime_type - external: ecs -- name: file.size - external: ecs -- name: network.community_id - external: ecs -- name: network.protocol - external: ecs -- name: related.ip - external: ecs -- name: related.user - external: ecs -- name: source.address - external: ecs -- name: source.as.number - external: ecs -- name: source.as.organization.name - external: ecs -- name: log.level - external: ecs -- name: source.geo.city_name - external: ecs -- name: source.geo.continent_name - external: ecs -- name: source.geo.country_iso_code - external: ecs -- name: source.geo.country_name - external: ecs -- name: source.geo.location - external: ecs -- name: source.geo.name - external: ecs -- name: source.geo.region_iso_code - external: ecs -- name: source.geo.region_name - external: ecs -- name: source.ip - external: ecs -- name: source.port - external: ecs -- name: user.full_name - external: ecs -- name: user.domain - external: ecs -- name: user.id - external: ecs -- name: user.name - external: ecs -- name: tags - external: ecs -- name: url.original - external: ecs -- name: url.domain - external: ecs -- name: rule.name - external: ecs diff --git a/packages/azure_frontdoor/docs/README.md b/packages/azure_frontdoor/docs/README.md index 09930e849bf..45b6d1c7253 100644 --- a/packages/azure_frontdoor/docs/README.md +++ b/packages/azure_frontdoor/docs/README.md @@ -73,107 +73,15 @@ Users can also use this in case of a Hybrid Cloud model, where one may define th | azure.frontdoor.operation_name | Azure operation name. | keyword | | azure.frontdoor.resource_id | Azure Resource ID. | keyword | | azure.frontdoor.tracking_reference | The unique reference string that identifies a request served by AFD, also sent as X-Azure-Ref header to the client. Required for searching details in the access logs for a specific request. | keyword | -| client.address | Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| client.port | Port of the client. | long | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.mime_type | MIME type should identify the format of the file or stream of bytes using https://www.iana.org/assignments/media-types/media-types.xhtml[IANA official types], where possible. When more than one type is applicable, the most specific type should be used. | keyword | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.bytes | Total size in bytes of the request (body and headers). | long | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.response.bytes | Total size in bytes of the response (body and headers). | long | -| http.response.status_code | HTTP response status code. | long | -| http.version | HTTP version. | keyword | | input.type | Input type. | keyword | -| log.file.path | Log file path. | keyword | -| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | | log.offset | Log offset. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| tls.version | Numeric part of the version parsed from the original string. | keyword | -| tls.version_protocol | Normalized lowercase protocol name parsed from original string. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | ## WAF Logs @@ -192,90 +100,13 @@ Users can also use this in case of a Hybrid Cloud model, where one may define th | azure.frontdoor.waf.policy | WAF policy name. | keyword | | azure.frontdoor.waf.policy_mode | WAF policy mode. | keyword | | azure.frontdoor.waf.time | The date and time when the AFD edge delivered requested contents to client (in UTC). | keyword | -| client.address | Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| client.port | Port of the client. | long | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.mime_type | MIME type should identify the format of the file or stream of bytes using https://www.iana.org/assignments/media-types/media-types.xhtml[IANA official types], where possible. When more than one type is applicable, the most specific type should be used. | keyword | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type. | keyword | -| log.file.path | Log file path. | keyword | -| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | | log.offset | Log offset. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | diff --git a/packages/azure_frontdoor/manifest.yml b/packages/azure_frontdoor/manifest.yml index 34427459aaa..2bf8e6c01d9 100644 --- a/packages/azure_frontdoor/manifest.yml +++ b/packages/azure_frontdoor/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: azure_frontdoor title: "Azure Frontdoor" -version: "1.7.0" +version: "1.8.0" description: "This Elastic integration collects logs from Azure Frontdoor." type: integration categories: @@ -11,7 +11,7 @@ categories: - web conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" screenshots: - src: /img/azure-frontdoor-overview.png title: Azure Frontdoor Overview From e86d8a38bfc27dd91715b92635ec7de060f8b595 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:08 +0930 Subject: [PATCH 011/121] [azure_network_watcher_nsg] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/azure_network_watcher_nsg --- packages/azure_network_watcher_nsg/_dev/build/build.yml | 1 - packages/azure_network_watcher_nsg/changelog.yml | 5 +++++ .../data_stream/log/fields/beats.yml | 3 --- packages/azure_network_watcher_nsg/docs/README.md | 1 - packages/azure_network_watcher_nsg/manifest.yml | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/azure_network_watcher_nsg/_dev/build/build.yml b/packages/azure_network_watcher_nsg/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/azure_network_watcher_nsg/_dev/build/build.yml +++ b/packages/azure_network_watcher_nsg/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/azure_network_watcher_nsg/changelog.yml b/packages/azure_network_watcher_nsg/changelog.yml index 1cc58ec7ce6..8fd6ed043a0 100644 --- a/packages/azure_network_watcher_nsg/changelog.yml +++ b/packages/azure_network_watcher_nsg/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.2.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: 0.1.0 changes: - description: Initial release. diff --git a/packages/azure_network_watcher_nsg/data_stream/log/fields/beats.yml b/packages/azure_network_watcher_nsg/data_stream/log/fields/beats.yml index 80cbae91cae..cc9fcebf29b 100644 --- a/packages/azure_network_watcher_nsg/data_stream/log/fields/beats.yml +++ b/packages/azure_network_watcher_nsg/data_stream/log/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/azure_network_watcher_nsg/docs/README.md b/packages/azure_network_watcher_nsg/docs/README.md index a6733bff235..6a39e15b95d 100644 --- a/packages/azure_network_watcher_nsg/docs/README.md +++ b/packages/azure_network_watcher_nsg/docs/README.md @@ -438,5 +438,4 @@ An example event for `log` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | diff --git a/packages/azure_network_watcher_nsg/manifest.yml b/packages/azure_network_watcher_nsg/manifest.yml index 1a820562ea3..304c80719b0 100644 --- a/packages/azure_network_watcher_nsg/manifest.yml +++ b/packages/azure_network_watcher_nsg/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.1.2 name: azure_network_watcher_nsg title: Azure Network Watcher NSG -version: 0.1.0 +version: "0.2.0" description: Collect logs from Azure Network Watcher NSG with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - security conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: subscription: basic screenshots: From 51a1920ebb1bbfba6a3f2a48d906f0226440a6dd Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:09 +0930 Subject: [PATCH 012/121] [azure_network_watcher_vnet] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/azure_network_watcher_vnet --- packages/azure_network_watcher_vnet/_dev/build/build.yml | 1 - packages/azure_network_watcher_vnet/changelog.yml | 5 +++++ .../data_stream/log/fields/beats.yml | 3 --- packages/azure_network_watcher_vnet/docs/README.md | 1 - packages/azure_network_watcher_vnet/manifest.yml | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/azure_network_watcher_vnet/_dev/build/build.yml b/packages/azure_network_watcher_vnet/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/azure_network_watcher_vnet/_dev/build/build.yml +++ b/packages/azure_network_watcher_vnet/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/azure_network_watcher_vnet/changelog.yml b/packages/azure_network_watcher_vnet/changelog.yml index 409cd975795..585b475a4c0 100644 --- a/packages/azure_network_watcher_vnet/changelog.yml +++ b/packages/azure_network_watcher_vnet/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.2.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: 0.1.0 changes: - description: Initial release. diff --git a/packages/azure_network_watcher_vnet/data_stream/log/fields/beats.yml b/packages/azure_network_watcher_vnet/data_stream/log/fields/beats.yml index 80cbae91cae..cc9fcebf29b 100644 --- a/packages/azure_network_watcher_vnet/data_stream/log/fields/beats.yml +++ b/packages/azure_network_watcher_vnet/data_stream/log/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/azure_network_watcher_vnet/docs/README.md b/packages/azure_network_watcher_vnet/docs/README.md index 69013cdbc26..7b6af8bc33f 100644 --- a/packages/azure_network_watcher_vnet/docs/README.md +++ b/packages/azure_network_watcher_vnet/docs/README.md @@ -804,5 +804,4 @@ An example event for `log` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | diff --git a/packages/azure_network_watcher_vnet/manifest.yml b/packages/azure_network_watcher_vnet/manifest.yml index d48d99ff2d8..9f673edf658 100644 --- a/packages/azure_network_watcher_vnet/manifest.yml +++ b/packages/azure_network_watcher_vnet/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.1.2 name: azure_network_watcher_vnet title: Azure Network Watcher VNet -version: 0.1.0 +version: "0.2.0" description: Collect logs from Azure Network Watcher VNet with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - security conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: subscription: basic screenshots: From 95d1079d2673eceaca6a69a54b2f55b521c28a32 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:11 +0930 Subject: [PATCH 013/121] [barracuda] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.4.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/barracuda --- packages/barracuda/changelog.yml | 5 + .../barracuda/data_stream/waf/fields/ecs.yml | 275 ------------------ packages/barracuda/docs/README.md | 150 ---------- packages/barracuda/manifest.yml | 4 +- 4 files changed, 7 insertions(+), 427 deletions(-) delete mode 100644 packages/barracuda/data_stream/waf/fields/ecs.yml diff --git a/packages/barracuda/changelog.yml b/packages/barracuda/changelog.yml index d4c288c8f0b..3a2b280f0d1 100644 --- a/packages/barracuda/changelog.yml +++ b/packages/barracuda/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.14.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.13.0" changes: - description: Make `host.ip` field conform to ECS field definition. diff --git a/packages/barracuda/data_stream/waf/fields/ecs.yml b/packages/barracuda/data_stream/waf/fields/ecs.yml deleted file mode 100644 index d88833d12a5..00000000000 --- a/packages/barracuda/data_stream/waf/fields/ecs.yml +++ /dev/null @@ -1,275 +0,0 @@ -- external: ecs - name: client.as.number -- external: ecs - name: client.as.organization.name -- external: ecs - name: client.geo.city_name -- external: ecs - name: client.geo.continent_name -- external: ecs - name: client.geo.country_iso_code -- external: ecs - name: client.geo.country_name -- external: ecs - name: client.geo.location -- external: ecs - name: client.geo.region_iso_code -- external: ecs - name: client.geo.region_name -- external: ecs - name: client.user.name -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.category -- external: ecs - name: event.code -- external: ecs - name: event.created -- external: ecs - name: event.duration -- external: ecs - name: event.end -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.provider -- external: ecs - name: event.severity -- external: ecs - name: event.start -- external: ecs - name: event.timezone -- external: ecs - name: event.type -- external: ecs - name: file.path -- external: ecs - name: labels -- external: ecs - name: http.request.id -- external: ecs - name: http.request.referrer -- external: ecs - name: http.request.method -- external: ecs - name: http.request.bytes -- external: ecs - name: http.response.bytes -- external: ecs - name: http.response.status_code -- external: ecs - name: http.version -- external: ecs - name: log.file.path -- external: ecs - name: log.level -- external: ecs - name: message -- external: ecs - name: network.bytes -- external: ecs - name: network.community_id -- external: ecs - name: network.direction -- external: ecs - name: network.forwarded_ip -- external: ecs - name: network.iana_number -- external: ecs - name: network.inner - type: group -- external: ecs - name: network.inner.vlan.id -- external: ecs - name: network.inner.vlan.name -- external: ecs - name: network.protocol -- external: ecs - name: network.transport -- external: ecs - name: network.type -- external: ecs - name: observer.egress.interface.name -- external: ecs - name: observer.egress.zone -- external: ecs - name: observer.hostname -- external: ecs - name: observer.ingress.interface.name -- external: ecs - name: observer.ingress.zone -- external: ecs - name: observer.ip -- external: ecs - name: observer.name -- external: ecs - name: observer.product -- external: ecs - name: observer.type -- external: ecs - name: observer.vendor -- external: ecs - name: observer.version -- external: ecs - name: process.name -- external: ecs - name: process.pid -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: rule.category -- external: ecs - name: rule.description -- external: ecs - name: rule.name -- external: ecs - name: source.address -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.bytes -- external: ecs - name: source.domain -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: source.nat.ip -- external: ecs - name: source.nat.port -- external: ecs - name: source.port -- external: ecs - name: source.user.domain -- external: ecs - name: source.user.name -- external: ecs - name: source.user.id -- external: ecs - name: source.user.group.name -- external: ecs - name: tags -- external: ecs - name: url.domain -- external: ecs - name: url.extension -- external: ecs - name: url.fragment -- external: ecs - name: url.full -- external: ecs - name: url.original -- external: ecs - name: url.password -- external: ecs - name: url.path -- external: ecs - name: url.port -- external: ecs - name: url.query -- external: ecs - name: url.registered_domain -- external: ecs - name: url.scheme -- external: ecs - name: url.subdomain -- external: ecs - name: url.top_level_domain -- external: ecs - name: user.id -- external: ecs - name: url.username -- external: ecs - name: user.email -- external: ecs - name: user.name -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version -- external: ecs - name: server.domain -- external: ecs - name: server.address -- external: ecs - name: server.port -- external: ecs - name: server.ip -- external: ecs - name: server.user.name -- external: ecs - name: server.geo.city_name -- external: ecs - name: server.geo.continent_name -- external: ecs - name: server.geo.country_iso_code -- external: ecs - name: server.geo.country_name -- external: ecs - name: server.geo.location -- external: ecs - name: server.geo.region_iso_code -- external: ecs - name: server.geo.region_name -- external: ecs - name: client.domain -- external: ecs - name: client.address -- external: ecs - name: client.port -- external: ecs - name: client.ip -- external: ecs - name: client.user.id diff --git a/packages/barracuda/docs/README.md b/packages/barracuda/docs/README.md index e9cc7d27498..087b6616139 100644 --- a/packages/barracuda/docs/README.md +++ b/packages/barracuda/docs/README.md @@ -133,160 +133,10 @@ An example event for `waf` looks as following: | barracuda.waf.unit_name | Specifies the name of the unit. | keyword | | barracuda.waf.user_id | The identifier of the user. | keyword | | barracuda.waf.wf_matched | Specifies whether the request is valid. Values:INVALID, VALID. | keyword | -| client.address | Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| client.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| client.as.organization.name | Organization name. | keyword | -| client.as.organization.name.text | Multi-field of `client.as.organization.name`. | match_only_text | -| client.domain | The domain name of the client system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| client.geo.city_name | City name. | keyword | -| client.geo.continent_name | Name of the continent. | keyword | -| client.geo.country_iso_code | Country ISO code. | keyword | -| client.geo.country_name | Country name. | keyword | -| client.geo.location | Longitude and latitude. | geo_point | -| client.geo.region_iso_code | Region ISO code. | keyword | -| client.geo.region_name | Region name. | keyword | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| client.port | Port of the client. | long | -| client.user.id | Unique identifier of the user. | keyword | -| client.user.name | Short name or login of the user. | keyword | -| client.user.name.text | Multi-field of `client.user.name`. | match_only_text | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | -| event.duration | Duration of the event in nanoseconds. If `event.start` and `event.end` are known this value should be the difference between the end and start time. | long | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| http.request.bytes | Total size in bytes of the request (body and headers). | long | -| http.request.id | A unique identifier for each HTTP request to correlate logs between clients and servers in transactions. The id may be contained in a non-standard HTTP header, such as `X-Request-ID` or `X-Correlation-ID`. | keyword | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.request.referrer | Referrer for this HTTP request. | keyword | -| http.response.bytes | Total size in bytes of the response (body and headers). | long | -| http.response.status_code | HTTP response status code. | long | -| http.version | HTTP version. | keyword | | input.type | Input type | keyword | -| labels | Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels. | object | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | -| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.bytes | Total bytes transferred in both directions. If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. | long | -| network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.forwarded_ip | Host IP address when the source IP address is the proxy. | ip | -| network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword | -| network.inner | Network.inner fields are added in addition to network.vlan fields to describe the innermost VLAN when q-in-q VLAN tagging is present. Allowed fields include vlan.id and vlan.name. Inner vlan fields are typically used when sending traffic with multiple 802.1q encapsulations to a network sensor (e.g. Zeek, Wireshark.) | group | -| network.inner.vlan.id | VLAN ID as reported by the observer. | keyword | -| network.inner.vlan.name | Optional VLAN name as reported by the observer. | keyword | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | -| observer.egress.interface.name | Interface name as reported by the system. | keyword | -| observer.egress.zone | Network zone of outbound traffic as reported by the observer to categorize the destination area of egress traffic, e.g. Internal, External, DMZ, HR, Legal, etc. | keyword | -| observer.hostname | Hostname of the observer. | keyword | -| observer.ingress.interface.name | Interface name as reported by the system. | keyword | -| observer.ingress.zone | Network zone of incoming traffic as reported by the observer to categorize the source area of ingress traffic. e.g. internal, External, DMZ, HR, Legal, etc. | keyword | -| observer.ip | IP addresses of the observer. | ip | -| observer.name | Custom name of the observer. This is a name that can be given to an observer. This can be helpful for example if multiple firewalls of the same model are used in an organization. If no custom name is needed, the field can be left empty. | keyword | -| observer.product | The product name of the observer. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| observer.version | Observer version. | keyword | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.pid | Process id. | long | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.category | A categorization value keyword used by the entity using the rule for detection of this event. | keyword | -| rule.description | The description of the rule generating the event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| server.address | Some event server addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| server.domain | The domain name of the server system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| server.geo.city_name | City name. | keyword | -| server.geo.continent_name | Name of the continent. | keyword | -| server.geo.country_iso_code | Country ISO code. | keyword | -| server.geo.country_name | Country name. | keyword | -| server.geo.location | Longitude and latitude. | geo_point | -| server.geo.region_iso_code | Region ISO code. | keyword | -| server.geo.region_name | Region name. | keyword | -| server.ip | IP address of the server (IPv4 or IPv6). | ip | -| server.port | Port of the server. | long | -| server.user.name | Short name or login of the user. | keyword | -| server.user.name.text | Multi-field of `server.user.name`. | match_only_text | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.nat.ip | Translated ip of source based NAT sessions (e.g. internal client to internet) Typically connections traversing load balancers, firewalls, or routers. | ip | -| source.nat.port | Translated port of source based NAT sessions. (e.g. internal client to internet) Typically used with load balancers, firewalls, or routers. | long | -| source.port | Port of the source. | long | -| source.user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| source.user.group.name | Name of the group. | keyword | -| source.user.id | Unique identifier of the user. | keyword | -| source.user.name | Short name or login of the user. | keyword | -| source.user.name.text | Multi-field of `source.user.name`. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | -| url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| url.full.text | Multi-field of `url.full`. | match_only_text | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.password | Password of the request. | keyword | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| url.username | Username of the request. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | diff --git a/packages/barracuda/manifest.yml b/packages/barracuda/manifest.yml index 4542a70db12..01c018f574c 100644 --- a/packages/barracuda/manifest.yml +++ b/packages/barracuda/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: barracuda title: "Barracuda Web Application Firewall" -version: "1.13.0" +version: "1.14.0" description: "Collect logs from Barracuda Web Application Firewall with Elastic Agent." type: integration source: @@ -12,7 +12,7 @@ categories: - web_application_firewall conditions: kibana: - version: ^8.4.0 + version: "^8.13.0" elastic: subscription: basic screenshots: From e20083606fcf7edbdaa73bdb13703f2e66bd08af Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:12 +0930 Subject: [PATCH 014/121] [barracuda_cloudgen_firewall] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.5.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/barracuda_cloudgen_firewall --- .../barracuda_cloudgen_firewall/changelog.yml | 5 + .../data_stream/log/fields/ecs.yml | 194 ------------------ .../docs/README.md | 106 ---------- .../barracuda_cloudgen_firewall/manifest.yml | 4 +- 4 files changed, 7 insertions(+), 302 deletions(-) diff --git a/packages/barracuda_cloudgen_firewall/changelog.yml b/packages/barracuda_cloudgen_firewall/changelog.yml index 8ea2464cbc3..45406c8cbcb 100644 --- a/packages/barracuda_cloudgen_firewall/changelog.yml +++ b/packages/barracuda_cloudgen_firewall/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.12.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.11.0" changes: - description: Update manifest format version to v3.0.3. diff --git a/packages/barracuda_cloudgen_firewall/data_stream/log/fields/ecs.yml b/packages/barracuda_cloudgen_firewall/data_stream/log/fields/ecs.yml index bf39f98bbf8..adb0dc85322 100644 --- a/packages/barracuda_cloudgen_firewall/data_stream/log/fields/ecs.yml +++ b/packages/barracuda_cloudgen_firewall/data_stream/log/fields/ecs.yml @@ -1,196 +1,2 @@ - external: ecs name: '@timestamp' -- external: ecs - name: destination.address -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.bytes -- external: ecs - name: destination.domain -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.geo.name -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.ip -- external: ecs - name: destination.mac -- external: ecs - name: destination.nat.ip -- external: ecs - name: destination.packets -- external: ecs - name: destination.port -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.code -- external: ecs - name: event.ingested -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.timezone -- external: ecs - name: http.request.body.bytes -- external: ecs - name: http.request.bytes -- external: ecs - name: http.request.method -- external: ecs - name: http.request.mime_type -- external: ecs - name: http.request.referrer -- external: ecs - name: http.response.body.bytes -- external: ecs - name: http.response.bytes -- external: ecs - name: http.response.status_code -- external: ecs - name: http.version -- external: ecs - name: labels -- external: ecs - name: log.level -- external: ecs - name: message -- external: ecs - name: network.community_id -- external: ecs - name: network.iana_number -- external: ecs - name: network.transport -- external: ecs - name: network.type -- external: ecs - name: observer.egress.interface.name -- external: ecs - name: observer.hostname -- external: ecs - name: observer.ingress.interface.name -- external: ecs - name: observer.product -- external: ecs - name: observer.serial_number -- external: ecs - name: observer.type -- external: ecs - name: observer.vendor -- external: ecs - name: observer.version -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: rule.category -- external: ecs - name: rule.description -- external: ecs - name: rule.name -- external: ecs - name: rule.ruleset -- external: ecs - name: source.address -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.bytes -- external: ecs - name: source.domain -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.name -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: source.mac -- external: ecs - name: source.nat.ip -- external: ecs - name: source.packets -- external: ecs - name: source.port -- external: ecs - name: tags -- external: ecs - name: url.domain -- external: ecs - name: url.extension -- external: ecs - name: url.full -- external: ecs - name: url.original -- external: ecs - name: url.password -- external: ecs - name: url.path -- external: ecs - name: url.port -- external: ecs - name: url.query -- external: ecs - name: url.scheme -- external: ecs - name: url.username -- external: ecs - name: user.domain -- external: ecs - name: user.full_name -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version diff --git a/packages/barracuda_cloudgen_firewall/docs/README.md b/packages/barracuda_cloudgen_firewall/docs/README.md index dd1860554b2..359fad18c1d 100644 --- a/packages/barracuda_cloudgen_firewall/docs/README.md +++ b/packages/barracuda_cloudgen_firewall/docs/README.md @@ -160,114 +160,8 @@ An example event for `log` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.bytes | Bytes sent from the destination to the source. | long | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.mac | MAC address of the destination. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| destination.nat.ip | Translated ip of destination based NAT sessions (e.g. internet to private DMZ) Typically used with load balancers, firewalls, or routers. | ip | -| destination.packets | Packets sent from the destination to the source. | long | -| destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| http.request.body.bytes | Size in bytes of the request body. | long | -| http.request.bytes | Total size in bytes of the request (body and headers). | long | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.request.mime_type | Mime type of the body of the request. This value must only be populated based on the content of the request body, not on the `Content-Type` header. Comparing the mime type of a request with the request's Content-Type header can be helpful in detecting threats or misconfigured clients. | keyword | -| http.request.referrer | Referrer for this HTTP request. | keyword | -| http.response.body.bytes | Size in bytes of the response body. | long | -| http.response.bytes | Total size in bytes of the response (body and headers). | long | -| http.response.status_code | HTTP response status code. | long | -| http.version | HTTP version. | keyword | | input.type | Type of Filebeat input. | keyword | -| labels | Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels. | object | | labels.origin_address | Remote address where the log originated. | keyword | | labels.origin_client_subject | Distinguished name of subject of the x.509 certificate presented by the origin client when mutual TLS is enabled. | keyword | -| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword | -| network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | -| observer.egress.interface.name | Interface name as reported by the system. | keyword | -| observer.hostname | Hostname of the observer. | keyword | -| observer.ingress.interface.name | Interface name as reported by the system. | keyword | -| observer.product | The product name of the observer. | keyword | -| observer.serial_number | Observer serial number. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| observer.version | Observer version. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.category | A categorization value keyword used by the entity using the rule for detection of this event. | keyword | -| rule.description | The description of the rule generating the event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| rule.ruleset | Name of the ruleset, policy, group, or parent category in which the rule used to generate this event is a member. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.mac | MAC address of the source. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| source.nat.ip | Translated ip of source based NAT sessions (e.g. internal client to internet) Typically connections traversing load balancers, firewalls, or routers. | ip | -| source.packets | Packets sent from the source to the destination. | long | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| url.full.text | Multi-field of `url.full`. | match_only_text | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.password | Password of the request. | keyword | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.username | Username of the request. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | diff --git a/packages/barracuda_cloudgen_firewall/manifest.yml b/packages/barracuda_cloudgen_firewall/manifest.yml index 9821b00bf6c..4dfe48e12c5 100644 --- a/packages/barracuda_cloudgen_firewall/manifest.yml +++ b/packages/barracuda_cloudgen_firewall/manifest.yml @@ -1,13 +1,13 @@ format_version: "3.0.3" name: barracuda_cloudgen_firewall title: Barracuda CloudGen Firewall Logs -version: "1.11.0" +version: "1.12.0" description: Collect logs from Barracuda CloudGen Firewall devices with Elastic Agent. categories: ["network", "security", "firewall_security"] type: integration conditions: kibana: - version: "^8.5.0" + version: "^8.13.0" policy_templates: - name: barracuda_cloudgen_firewall title: Barracuda CloudGen Firewall Logs From 5d26f7268dad7eb4f2bb50ca4b38df0dbf6c1e72 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:13 +0930 Subject: [PATCH 015/121] [bbot] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.1 to ^8.13.0. The set ecs.version processor in pipelines was changed 8.11.0. Previously the pipeline was setting version 8.12.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. The ecs.version in sample_event.json files was changed to 8.11.0. Previously sample_event.json files contained 8.12.0. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/bbot --- packages/bbot/_dev/build/build.yml | 1 - packages/bbot/changelog.yml | 5 + .../test-bbot-ndjson.log-expected.json | 20 +- .../elasticsearch/ingest_pipeline/default.yml | 2 +- .../data_stream/asm_intel/fields/beats.yml | 3 - .../bbot/data_stream/asm_intel/fields/ecs.yml | 16 - .../data_stream/asm_intel/fields/fields.yml | 9 +- .../data_stream/asm_intel/sample_event.json | 2 +- packages/bbot/docs/README.md | 12 +- ...-8abcb381-42b3-4d99-a177-c103255eedd9.json | 2528 ++++++++--------- ...-45ce1599-99e3-4c4e-9c1a-07254be0e274.json | 216 +- packages/bbot/manifest.yml | 4 +- 12 files changed, 1395 insertions(+), 1423 deletions(-) delete mode 100644 packages/bbot/data_stream/asm_intel/fields/ecs.yml diff --git a/packages/bbot/_dev/build/build.yml b/packages/bbot/_dev/build/build.yml index 1f4fa988f6e..e2b012548e0 100644 --- a/packages/bbot/_dev/build/build.yml +++ b/packages/bbot/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: git@v8.11.0 - import_mappings: true diff --git a/packages/bbot/changelog.yml b/packages/bbot/changelog.yml index bdb726760d2..dfb40ebde34 100644 --- a/packages/bbot/changelog.yml +++ b/packages/bbot/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.2.0" + changes: + - description: ECS version updated to 8.11.0. Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "0.1.0" changes: - description: Initial release of the bbot package diff --git a/packages/bbot/data_stream/asm_intel/_dev/test/pipeline/test-bbot-ndjson.log-expected.json b/packages/bbot/data_stream/asm_intel/_dev/test/pipeline/test-bbot-ndjson.log-expected.json index f97a322a430..935393ad03e 100644 --- a/packages/bbot/data_stream/asm_intel/_dev/test/pipeline/test-bbot-ndjson.log-expected.json +++ b/packages/bbot/data_stream/asm_intel/_dev/test/pipeline/test-bbot-ndjson.log-expected.json @@ -25,7 +25,7 @@ "type": "DNS_NAME" }, "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "kind": "asset" @@ -67,7 +67,7 @@ "type": "ORG_STUB" }, "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "kind": "asset" @@ -100,7 +100,7 @@ "type": "PROTOCOL" }, "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "kind": "asset" @@ -142,7 +142,7 @@ "type": "DNS_NAME" }, "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "kind": "asset" @@ -188,7 +188,7 @@ "type": "TECHNOLOGY" }, "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "kind": "asset" @@ -235,7 +235,7 @@ "type": "FINDING" }, "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "kind": "asset" @@ -276,7 +276,7 @@ "type": "PROTOCOL" }, "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "kind": "asset" @@ -322,7 +322,7 @@ "type": "ASN" }, "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "kind": "asset" @@ -355,7 +355,7 @@ "web_spider_distance": 0 }, "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "kind": "asset" @@ -408,7 +408,7 @@ "type": "WAF" }, "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "kind": "asset" diff --git a/packages/bbot/data_stream/asm_intel/elasticsearch/ingest_pipeline/default.yml b/packages/bbot/data_stream/asm_intel/elasticsearch/ingest_pipeline/default.yml index a047fcb7674..4971e4b7a93 100644 --- a/packages/bbot/data_stream/asm_intel/elasticsearch/ingest_pipeline/default.yml +++ b/packages/bbot/data_stream/asm_intel/elasticsearch/ingest_pipeline/default.yml @@ -7,7 +7,7 @@ processors: #################### - set: field: ecs.version - value: 8.12.0 + value: 8.11.0 - set: field: event.kind value: asset diff --git a/packages/bbot/data_stream/asm_intel/fields/beats.yml b/packages/bbot/data_stream/asm_intel/fields/beats.yml index 3415608ae37..cc9fcebf29b 100644 --- a/packages/bbot/data_stream/asm_intel/fields/beats.yml +++ b/packages/bbot/data_stream/asm_intel/fields/beats.yml @@ -1,9 +1,6 @@ - name: input.type description: Type of Filebeat input. type: keyword -- name: tags - type: keyword - description: User defined tags. - name: log.offset type: long description: Log offset. diff --git a/packages/bbot/data_stream/asm_intel/fields/ecs.yml b/packages/bbot/data_stream/asm_intel/fields/ecs.yml deleted file mode 100644 index 5f506ad1f66..00000000000 --- a/packages/bbot/data_stream/asm_intel/fields/ecs.yml +++ /dev/null @@ -1,16 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.kind -- external: ecs - name: message -- external: ecs - name: event.original -- external: ecs - name: host.ip -- external: ecs - name: url.port -- external: ecs - name: vulnerability.severity -- external: ecs - name: url.full diff --git a/packages/bbot/data_stream/asm_intel/fields/fields.yml b/packages/bbot/data_stream/asm_intel/fields/fields.yml index 3e0a5ae571d..e16a006d1b1 100644 --- a/packages/bbot/data_stream/asm_intel/fields/fields.yml +++ b/packages/bbot/data_stream/asm_intel/fields/fields.yml @@ -133,16 +133,12 @@ - name: source type: keyword - - name: tags type: keyword - - name: timestamp type: date - - name: type type: keyword - - name: web_spider_distance type: integer description: > @@ -163,12 +159,12 @@ description: > Description of the asn. - - name: data.asn.name + - name: data.asn.name type: keyword description: > Name discovered for the asn. - - name: data.asn.subnet + - name: data.asn.subnet type: keyword description: > Subnet discovered for the asn. @@ -242,3 +238,4 @@ type: keyword description: > URL of the data finding. + diff --git a/packages/bbot/data_stream/asm_intel/sample_event.json b/packages/bbot/data_stream/asm_intel/sample_event.json index f1a0bd9fb3b..76fece33b85 100644 --- a/packages/bbot/data_stream/asm_intel/sample_event.json +++ b/packages/bbot/data_stream/asm_intel/sample_event.json @@ -29,7 +29,7 @@ "type": "logs" }, "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "elastic_agent": { "id": "bcb4b946-41b8-4916-9308-849b3bf23f46", diff --git a/packages/bbot/docs/README.md b/packages/bbot/docs/README.md index 850520ca77b..5c6001af9c4 100644 --- a/packages/bbot/docs/README.md +++ b/packages/bbot/docs/README.md @@ -66,7 +66,7 @@ An example event for `asm_intel` looks as following: "type": "logs" }, "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "elastic_agent": { "id": "bcb4b946-41b8-4916-9308-849b3bf23f46", @@ -167,16 +167,6 @@ An example event for `asm_intel` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| host.ip | Host ip addresses. | ip | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| tags | User defined tags. | keyword | -| url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| url.full.text | Multi-field of `url.full`. | match_only_text | -| url.port | Port of the request, such as 443. | long | -| vulnerability.severity | The severity of the vulnerability can help with metrics and internal prioritization regarding remediation. For example (https://nvd.nist.gov/vuln-metrics/cvss) | keyword | diff --git a/packages/bbot/kibana/dashboard/bbot-8abcb381-42b3-4d99-a177-c103255eedd9.json b/packages/bbot/kibana/dashboard/bbot-8abcb381-42b3-4d99-a177-c103255eedd9.json index 4b368794744..40c32b82e5e 100644 --- a/packages/bbot/kibana/dashboard/bbot-8abcb381-42b3-4d99-a177-c103255eedd9.json +++ b/packages/bbot/kibana/dashboard/bbot-8abcb381-42b3-4d99-a177-c103255eedd9.json @@ -1,1265 +1,1265 @@ -{ - "attributes": { - "controlGroupInput": { - "chainingSystem": "HIERARCHICAL", - "controlStyle": "oneLine", - "ignoreParentSettingsJSON": "{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}", - "panelsJSON": "{\"7b900e62-ba4a-468b-a99f-aa5bf4a3a526\":{\"type\":\"optionsListControl\",\"order\":0,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"7b900e62-ba4a-468b-a99f-aa5bf4a3a526\",\"fieldName\":\"bbot.scan\",\"title\":\"Scan ID:\",\"grow\":true,\"width\":\"medium\",\"selectedOptions\":[],\"enhancements\":{}}}}" - }, - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } - }, - "optionsJSON": { - "hidePanelTitles": false, - "syncColors": false, - "syncCursor": true, - "syncTooltips": false, - "useMargins": true - }, - "panelsJSON": [ - { - "embeddableConfig": { - "attributes": { - "references": [ - { - "id": "logs-*", - "name": "indexpattern-datasource-layer-b041b892-4b58-48f3-9f5e-52e0e604cfb0", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "2604eb17-0109-4f38-993e-ed797031d791", - "type": "index-pattern" - } - ], - "state": { - "adHocDataViews": {}, - "datasourceStates": { - "formBased": { - "layers": { - "b041b892-4b58-48f3-9f5e-52e0e604cfb0": { - "columnOrder": [ - "b77c2eee-54f7-4fa0-9aa6-936d9064ff4f", - "436a5f51-90a1-4193-b109-25b90ab29fb0" - ], - "columns": { - "436a5f51-90a1-4193-b109-25b90ab29fb0": { - "customLabel": true, - "dataType": "number", - "isBucketed": false, - "label": "Records", - "operationType": "count", - "params": { - "emptyAsNull": true - }, - "scale": "ratio", - "sourceField": "___records___" - }, - "b77c2eee-54f7-4fa0-9aa6-936d9064ff4f": { - "customLabel": true, - "dataType": "date", - "isBucketed": true, - "label": "Date of scan", - "operationType": "date_histogram", - "params": { - "dropPartials": false, - "includeEmptyRows": true, - "interval": "1w" - }, - "scale": "interval", - "sourceField": "@timestamp" - } - }, - "incompleteColumns": {} - } - } - }, - "indexpattern": { - "layers": {} - }, - "textBased": { - "layers": {} - } - }, - "filters": [ - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "index": "2604eb17-0109-4f38-993e-ed797031d791", - "key": "event.dataset", - "negate": false, - "params": { - "query": "bbot.asm_intel" - }, - "type": "phrase" - }, - "query": { - "match_phrase": { - "event.dataset": "bbot.asm_intel" - } - } - } - ], - "internalReferences": [], - "query": { - "language": "kuery", - "query": "" - }, - "visualization": { - "axisTitlesVisibilitySettings": { - "x": true, - "yLeft": true, - "yRight": true - }, - "fittingFunction": "None", - "gridlinesVisibilitySettings": { - "x": true, - "yLeft": true, - "yRight": true - }, - "labelsOrientation": { - "x": 0, - "yLeft": 0, - "yRight": 0 - }, - "layers": [ - { - "accessors": [ - "436a5f51-90a1-4193-b109-25b90ab29fb0" - ], - "layerId": "b041b892-4b58-48f3-9f5e-52e0e604cfb0", - "layerType": "data", - "seriesType": "bar", - "xAccessor": "b77c2eee-54f7-4fa0-9aa6-936d9064ff4f", - "yConfig": [ - { - "color": "#e7664c", - "forAccessor": "436a5f51-90a1-4193-b109-25b90ab29fb0" - } - ] - } - ], - "legend": { - "isVisible": true, - "position": "right", - "showSingleSeries": false - }, - "preferredSeriesType": "bar", - "tickLabelsVisibilitySettings": { - "x": true, - "yLeft": true, - "yRight": true - }, - "valueLabels": "show" - } - }, - "title": "", - "type": "lens", - "visualizationType": "lnsXY" - }, - "enhancements": {}, - "hidePanelTitles": false, - "timeRange": { - "from": "now-2y", - "to": "now" - } - }, - "gridData": { - "h": 6, - "i": "ff18251e-b13b-42f6-8a10-6a6e61e2e74a", - "w": 48, - "x": 0, - "y": 0 - }, - "panelIndex": "ff18251e-b13b-42f6-8a10-6a6e61e2e74a", - "title": "Scans over time", - "type": "lens" - }, - { - "embeddableConfig": { - "attributes": { - "description": "This is a count of all url.domains found. There is some overlap between this field and the host.name field.", - "references": [ - { - "id": "logs-*", - "name": "indexpattern-datasource-layer-34706177-15e3-422e-942e-450494312e3f", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "d7a416f6-fbb4-4477-8760-363e18f9554c", - "type": "index-pattern" - } - ], - "state": { - "adHocDataViews": {}, - "datasourceStates": { - "formBased": { - "layers": { - "34706177-15e3-422e-942e-450494312e3f": { - "columnOrder": [ - "8847f861-0519-4914-b269-405389c0df68" - ], - "columns": { - "8847f861-0519-4914-b269-405389c0df68": { - "customLabel": true, - "dataType": "number", - "isBucketed": false, - "label": "Unique Records of Domain", - "operationType": "unique_count", - "params": { - "emptyAsNull": true - }, - "scale": "ratio", - "sourceField": "url.domain" - } - }, - "incompleteColumns": {} - } - } - }, - "indexpattern": { - "layers": {} - }, - "textBased": { - "layers": {} - } - }, - "filters": [ - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "index": "d7a416f6-fbb4-4477-8760-363e18f9554c", - "key": "event.dataset", - "negate": false, - "params": { - "query": "bbot.asm_intel" - }, - "type": "phrase" - }, - "query": { - "match_phrase": { - "event.dataset": "bbot.asm_intel" - } - } - } - ], - "internalReferences": [], - "query": { - "language": "kuery", - "query": "" - }, - "visualization": { - "color": "#E7664C", - "layerId": "34706177-15e3-422e-942e-450494312e3f", - "layerType": "data", - "metricAccessor": "8847f861-0519-4914-b269-405389c0df68" - } - }, - "title": "", - "type": "lens", - "visualizationType": "lnsMetric" - }, - "description": "This is a count of all url.domains found. There is some overlap between this field and the host.name field.", - "enhancements": {}, - "hidePanelTitles": false - }, - "gridData": { - "h": 5, - "i": "e2b473cb-83a3-43b9-9845-01a865ebba81", - "w": 15, - "x": 0, - "y": 6 - }, - "panelIndex": "e2b473cb-83a3-43b9-9845-01a865ebba81", - "title": "Unique Domains Found ", - "type": "lens" - }, - { - "embeddableConfig": { - "attributes": { - "description": "This is a count of all related.hosts found. This field contains IPv4, IPv6 and Domain Names. ", - "references": [ - { - "id": "logs-*", - "name": "indexpattern-datasource-layer-34706177-15e3-422e-942e-450494312e3f", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "ceb45dbd-8837-4fae-884c-5eef1f068cd9", - "type": "index-pattern" - } - ], - "state": { - "adHocDataViews": {}, - "datasourceStates": { - "formBased": { - "layers": { - "34706177-15e3-422e-942e-450494312e3f": { - "columnOrder": [ - "8847f861-0519-4914-b269-405389c0df68" - ], - "columns": { - "8847f861-0519-4914-b269-405389c0df68": { - "customLabel": true, - "dataType": "number", - "isBucketed": false, - "label": "Records Found for Related Hosts", - "operationType": "count", - "params": { - "emptyAsNull": true - }, - "scale": "ratio", - "sourceField": "related.hosts" - } - }, - "incompleteColumns": {} - } - } - }, - "indexpattern": { - "layers": {} - }, - "textBased": { - "layers": {} - } - }, - "filters": [ - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "index": "ceb45dbd-8837-4fae-884c-5eef1f068cd9", - "key": "event.dataset", - "negate": false, - "params": { - "query": "bbot.asm_intel" - }, - "type": "phrase" - }, - "query": { - "match_phrase": { - "event.dataset": "bbot.asm_intel" - } - } - } - ], - "internalReferences": [], - "query": { - "language": "kuery", - "query": "" - }, - "visualization": { - "color": "#E7664C", - "layerId": "34706177-15e3-422e-942e-450494312e3f", - "layerType": "data", - "metricAccessor": "8847f861-0519-4914-b269-405389c0df68" - } - }, - "title": "", - "type": "lens", - "visualizationType": "lnsMetric" - }, - "description": "This is a count of all related.hosts found. This field contains IPv4, IPv6 and Domain Names. ", - "enhancements": {}, - "hidePanelTitles": false - }, - "gridData": { - "h": 5, - "i": "8d154799-5342-4d9f-931a-8ac541b10235", - "w": 15, - "x": 15, - "y": 6 - }, - "panelIndex": "8d154799-5342-4d9f-931a-8ac541b10235", - "title": "Related Hosts Found - Count of Records", - "type": "lens" - }, - { - "embeddableConfig": { - "attributes": { - "references": [ - { - "id": "logs-*", - "name": "indexpattern-datasource-layer-9236266e-4c6d-4cb0-8d5c-49493bf23532", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "52db8b89-498c-4aa2-ba42-d65b2025598f", - "type": "index-pattern" - } - ], - "state": { - "adHocDataViews": {}, - "datasourceStates": { - "formBased": { - "layers": { - "9236266e-4c6d-4cb0-8d5c-49493bf23532": { - "columnOrder": [ - "0896481f-8b3d-45f6-bb23-665ece65f846", - "8be8fd12-8e1b-45d8-93e5-3903ae887fc8" - ], - "columns": { - "0896481f-8b3d-45f6-bb23-665ece65f846": { - "customLabel": true, - "dataType": "string", - "isBucketed": true, - "label": "Module", - "operationType": "terms", - "params": { - "exclude": [], - "excludeIsRegex": false, - "include": [], - "includeIsRegex": false, - "missingBucket": false, - "orderBy": { - "columnId": "8be8fd12-8e1b-45d8-93e5-3903ae887fc8", - "type": "column" - }, - "orderDirection": "desc", - "otherBucket": false, - "parentFormat": { - "id": "terms" - }, - "size": 5 - }, - "scale": "ordinal", - "sourceField": "bbot.module" - }, - "8be8fd12-8e1b-45d8-93e5-3903ae887fc8": { - "dataType": "number", - "isBucketed": false, - "label": "Count of records", - "operationType": "count", - "params": { - "emptyAsNull": true - }, - "scale": "ratio", - "sourceField": "___records___" - } - }, - "incompleteColumns": {}, - "sampling": 1 - } - } - }, - "indexpattern": { - "layers": {} - }, - "textBased": { - "layers": {} - } - }, - "filters": [ - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "index": "52db8b89-498c-4aa2-ba42-d65b2025598f", - "key": "event.dataset", - "negate": false, - "params": { - "query": "bbot.asm_intel" - }, - "type": "phrase" - }, - "query": { - "match_phrase": { - "event.dataset": "bbot.asm_intel" - } - } - } - ], - "internalReferences": [], - "query": { - "language": "kuery", - "query": "" - }, - "visualization": { - "axisTitlesVisibilitySettings": { - "x": true, - "yLeft": true, - "yRight": true - }, - "fittingFunction": "None", - "gridlinesVisibilitySettings": { - "x": true, - "yLeft": true, - "yRight": true - }, - "labelsOrientation": { - "x": 0, - "yLeft": 0, - "yRight": 0 - }, - "layers": [ - { - "accessors": [ - "8be8fd12-8e1b-45d8-93e5-3903ae887fc8" - ], - "layerId": "9236266e-4c6d-4cb0-8d5c-49493bf23532", - "layerType": "data", - "position": "top", - "seriesType": "bar_horizontal", - "showGridlines": false, - "xAccessor": "0896481f-8b3d-45f6-bb23-665ece65f846", - "yConfig": [ - { - "color": "#e7664c", - "forAccessor": "8be8fd12-8e1b-45d8-93e5-3903ae887fc8" - } - ] - } - ], - "legend": { - "isVisible": true, - "position": "right" - }, - "preferredSeriesType": "bar_horizontal", - "tickLabelsVisibilitySettings": { - "x": true, - "yLeft": true, - "yRight": true - }, - "valueLabels": "hide" - } - }, - "title": "", - "type": "lens", - "visualizationType": "lnsXY" - }, - "enhancements": {}, - "hidePanelTitles": false - }, - "gridData": { - "h": 21, - "i": "fd6001b7-89f1-4008-b56e-9fee8d3111b1", - "w": 18, - "x": 30, - "y": 6 - }, - "panelIndex": "fd6001b7-89f1-4008-b56e-9fee8d3111b1", - "title": "Popular Module Findings", - "type": "lens" - }, - { - "embeddableConfig": { - "attributes": { - "references": [ - { - "id": "logs-*", - "name": "indexpattern-datasource-layer-934f50cd-f1e9-47ea-be3a-3ceff354f1ad", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "1877a3bb-aa1f-420e-ad3b-b82ad23d1f0a", - "type": "index-pattern" - } - ], - "state": { - "adHocDataViews": {}, - "datasourceStates": { - "formBased": { - "layers": { - "934f50cd-f1e9-47ea-be3a-3ceff354f1ad": { - "columnOrder": [ - "eebfb3a7-f1b5-4ca3-97e0-95eb896f8621", - "f4935493-86bc-4383-b231-651c7b375e59" - ], - "columns": { - "eebfb3a7-f1b5-4ca3-97e0-95eb896f8621": { - "dataType": "string", - "isBucketed": true, - "label": "Top 5 values of url.domain", - "operationType": "terms", - "params": { - "exclude": [], - "excludeIsRegex": false, - "include": [], - "includeIsRegex": false, - "missingBucket": false, - "orderBy": { - "columnId": "f4935493-86bc-4383-b231-651c7b375e59", - "type": "column" - }, - "orderDirection": "desc", - "otherBucket": false, - "parentFormat": { - "id": "terms" - }, - "secondaryFields": [], - "size": 5 - }, - "scale": "ordinal", - "sourceField": "url.domain" - }, - "f4935493-86bc-4383-b231-651c7b375e59": { - "dataType": "number", - "isBucketed": false, - "label": "Count of records", - "operationType": "count", - "params": { - "emptyAsNull": true - }, - "scale": "ratio", - "sourceField": "___records___" - } - }, - "incompleteColumns": {}, - "sampling": 1 - } - } - }, - "indexpattern": { - "layers": {} - }, - "textBased": { - "layers": {} - } - }, - "filters": [ - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "index": "1877a3bb-aa1f-420e-ad3b-b82ad23d1f0a", - "key": "event.dataset", - "negate": false, - "params": { - "query": "bbot.asm_intel" - }, - "type": "phrase" - }, - "query": { - "match_phrase": { - "event.dataset": "bbot.asm_intel" - } - } - } - ], - "internalReferences": [], - "query": { - "language": "kuery", - "query": "" - }, - "visualization": { - "layers": [ - { - "categoryDisplay": "default", - "emptySizeRatio": 0.54, - "layerId": "934f50cd-f1e9-47ea-be3a-3ceff354f1ad", - "layerType": "data", - "legendDisplay": "default", - "metrics": [ - "f4935493-86bc-4383-b231-651c7b375e59" - ], - "nestedLegend": false, - "numberDisplay": "percent", - "primaryGroups": [ - "eebfb3a7-f1b5-4ca3-97e0-95eb896f8621" - ] - } - ], - "palette": { - "name": "negative", - "type": "palette" - }, - "shape": "donut" - } - }, - "title": "", - "type": "lens", - "visualizationType": "lnsPie" - }, - "enhancements": {}, - "hidePanelTitles": false - }, - "gridData": { - "h": 16, - "i": "ec50cd13-16ea-463b-8677-d6fc126fcaf8", - "w": 15, - "x": 0, - "y": 11 - }, - "panelIndex": "ec50cd13-16ea-463b-8677-d6fc126fcaf8", - "title": "Top 5 Domain Records", - "type": "lens" - }, - { - "embeddableConfig": { - "attributes": { - "description": "", - "references": [ - { - "id": "logs-*", - "name": "indexpattern-datasource-layer-934f50cd-f1e9-47ea-be3a-3ceff354f1ad", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "e6909ac9-f732-4420-a24d-69ffc4fe319c", - "type": "index-pattern" - } - ], - "state": { - "adHocDataViews": {}, - "datasourceStates": { - "formBased": { - "layers": { - "934f50cd-f1e9-47ea-be3a-3ceff354f1ad": { - "columnOrder": [ - "eebfb3a7-f1b5-4ca3-97e0-95eb896f8621", - "f4935493-86bc-4383-b231-651c7b375e59" - ], - "columns": { - "eebfb3a7-f1b5-4ca3-97e0-95eb896f8621": { - "dataType": "string", - "isBucketed": true, - "label": "Top 5 values of related.hosts", - "operationType": "terms", - "params": { - "exclude": [], - "excludeIsRegex": false, - "include": [], - "includeIsRegex": false, - "missingBucket": false, - "orderBy": { - "columnId": "f4935493-86bc-4383-b231-651c7b375e59", - "type": "column" - }, - "orderDirection": "desc", - "otherBucket": false, - "parentFormat": { - "id": "terms" - }, - "size": 5 - }, - "scale": "ordinal", - "sourceField": "related.hosts" - }, - "f4935493-86bc-4383-b231-651c7b375e59": { - "dataType": "number", - "isBucketed": false, - "label": "Count of records", - "operationType": "count", - "params": { - "emptyAsNull": true - }, - "scale": "ratio", - "sourceField": "___records___" - } - }, - "incompleteColumns": {}, - "sampling": 1 - } - } - }, - "indexpattern": { - "layers": {} - }, - "textBased": { - "layers": {} - } - }, - "filters": [ - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "index": "e6909ac9-f732-4420-a24d-69ffc4fe319c", - "key": "event.dataset", - "negate": false, - "params": { - "query": "bbot.asm_intel" - }, - "type": "phrase" - }, - "query": { - "match_phrase": { - "event.dataset": "bbot.asm_intel" - } - } - } - ], - "internalReferences": [], - "query": { - "language": "kuery", - "query": "" - }, - "visualization": { - "layers": [ - { - "categoryDisplay": "default", - "emptySizeRatio": 0.54, - "layerId": "934f50cd-f1e9-47ea-be3a-3ceff354f1ad", - "layerType": "data", - "legendDisplay": "default", - "metrics": [ - "f4935493-86bc-4383-b231-651c7b375e59" - ], - "nestedLegend": false, - "numberDisplay": "percent", - "primaryGroups": [ - "eebfb3a7-f1b5-4ca3-97e0-95eb896f8621" - ] - } - ], - "palette": { - "name": "negative", - "type": "palette" - }, - "shape": "donut" - } - }, - "title": "", - "type": "lens", - "visualizationType": "lnsPie" - }, - "description": "", - "enhancements": {}, - "hidePanelTitles": false - }, - "gridData": { - "h": 16, - "i": "b7513787-adcc-4e88-8211-42e9c559f09c", - "w": 15, - "x": 15, - "y": 11 - }, - "panelIndex": "b7513787-adcc-4e88-8211-42e9c559f09c", - "title": "Top 5 Related Hosts Found", - "type": "lens" - }, - { - "embeddableConfig": { - "enhancements": {} - }, - "gridData": { - "h": 21, - "i": "81963b3c-596f-4008-80de-286537f0c45d", - "w": 30, - "x": 0, - "y": 27 - }, - "panelIndex": "81963b3c-596f-4008-80de-286537f0c45d", - "panelRefName": "panel_81963b3c-596f-4008-80de-286537f0c45d", - "type": "search" - }, - { - "embeddableConfig": { - "attributes": { - "description": "", - "references": [ - { - "id": "logs-*", - "name": "indexpattern-datasource-layer-34706177-15e3-422e-942e-450494312e3f", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "f6dc81d5-5b2a-40b4-b17a-2b8034ac3bb0", - "type": "index-pattern" - } - ], - "state": { - "adHocDataViews": {}, - "datasourceStates": { - "formBased": { - "layers": { - "34706177-15e3-422e-942e-450494312e3f": { - "columnOrder": [ - "8847f861-0519-4914-b269-405389c0df68" - ], - "columns": { - "8847f861-0519-4914-b269-405389c0df68": { - "customLabel": true, - "dataType": "number", - "filter": { - "language": "kuery", - "query": "vulnerability.severity : * " - }, - "isBucketed": false, - "label": "Hosts found with Vulnerabilities", - "operationType": "unique_count", - "params": { - "emptyAsNull": true - }, - "scale": "ratio", - "sourceField": "url.domain" - } - }, - "incompleteColumns": {} - } - } - }, - "indexpattern": { - "layers": {} - }, - "textBased": { - "layers": {} - } - }, - "filters": [ - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "index": "f6dc81d5-5b2a-40b4-b17a-2b8034ac3bb0", - "key": "event.dataset", - "negate": false, - "params": { - "query": "bbot.asm_intel" - }, - "type": "phrase" - }, - "query": { - "match_phrase": { - "event.dataset": "bbot.asm_intel" - } - } - } - ], - "internalReferences": [], - "query": { - "language": "kuery", - "query": "" - }, - "visualization": { - "color": "#ffffff", - "layerId": "34706177-15e3-422e-942e-450494312e3f", - "layerType": "data", - "metricAccessor": "8847f861-0519-4914-b269-405389c0df68" - } - }, - "title": "", - "type": "lens", - "visualizationType": "lnsMetric" - }, - "description": "", - "enhancements": {}, - "hidePanelTitles": false - }, - "gridData": { - "h": 6, - "i": "17ab65a3-eb4a-47df-8e8c-91c8ca504c67", - "w": 18, - "x": 30, - "y": 27 - }, - "panelIndex": "17ab65a3-eb4a-47df-8e8c-91c8ca504c67", - "title": "Vulnerable Hosts", - "type": "lens" - }, - { - "embeddableConfig": { - "attributes": { - "references": [ - { - "id": "logs-*", - "name": "indexpattern-datasource-layer-34706177-15e3-422e-942e-450494312e3f", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "34e57322-6c1b-479e-95aa-318340186b2f", - "type": "index-pattern" - } - ], - "state": { - "adHocDataViews": {}, - "datasourceStates": { - "formBased": { - "layers": { - "34706177-15e3-422e-942e-450494312e3f": { - "columnOrder": [ - "2170eae6-6ab4-4fce-ac60-fbbd4301da66", - "b6a09dd7-f423-43e6-8068-db01ebfa9855" - ], - "columns": { - "2170eae6-6ab4-4fce-ac60-fbbd4301da66": { - "dataType": "string", - "isBucketed": true, - "label": "Top 5 values of vulnerability.severity", - "operationType": "terms", - "params": { - "exclude": [], - "excludeIsRegex": false, - "include": [], - "includeIsRegex": false, - "missingBucket": false, - "orderAgg": { - "dataType": "number", - "isBucketed": false, - "label": "Count of records", - "operationType": "count", - "params": { - "emptyAsNull": true - }, - "scale": "ratio", - "sourceField": "___records___" - }, - "orderBy": { - "type": "custom" - }, - "orderDirection": "desc", - "otherBucket": true, - "parentFormat": { - "id": "terms" - }, - "size": 5 - }, - "scale": "ordinal", - "sourceField": "vulnerability.severity" - }, - "b6a09dd7-f423-43e6-8068-db01ebfa9855": { - "customLabel": true, - "dataType": "number", - "isBucketed": false, - "label": "Severity Percentage", - "operationType": "count", - "params": { - "emptyAsNull": true - }, - "scale": "ratio", - "sourceField": "vulnerability.severity" - } - }, - "incompleteColumns": {} - } - } - }, - "indexpattern": { - "layers": {} - }, - "textBased": { - "layers": {} - } - }, - "filters": [ - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "index": "34e57322-6c1b-479e-95aa-318340186b2f", - "key": "event.dataset", - "negate": false, - "params": { - "query": "bbot.asm_intel" - }, - "type": "phrase" - }, - "query": { - "match_phrase": { - "event.dataset": "bbot.asm_intel" - } - } - } - ], - "internalReferences": [], - "query": { - "language": "kuery", - "query": "" - }, - "visualization": { - "axisTitlesVisibilitySettings": { - "x": true, - "yLeft": true, - "yRight": true - }, - "fittingFunction": "None", - "gridlinesVisibilitySettings": { - "x": true, - "yLeft": true, - "yRight": true - }, - "labelsOrientation": { - "x": 0, - "yLeft": 0, - "yRight": 0 - }, - "layers": [ - { - "accessors": [ - "b6a09dd7-f423-43e6-8068-db01ebfa9855" - ], - "layerId": "34706177-15e3-422e-942e-450494312e3f", - "layerType": "data", - "palette": { - "name": "negative", - "type": "palette" - }, - "seriesType": "bar_percentage_stacked", - "splitAccessor": "2170eae6-6ab4-4fce-ac60-fbbd4301da66" - } - ], - "legend": { - "isVisible": true, - "position": "right" - }, - "preferredSeriesType": "bar_percentage_stacked", - "tickLabelsVisibilitySettings": { - "x": true, - "yLeft": true, - "yRight": true - }, - "valueLabels": "show" - } - }, - "title": "", - "type": "lens", - "visualizationType": "lnsXY" - }, - "enhancements": {}, - "hidePanelTitles": false - }, - "gridData": { - "h": 15, - "i": "a45187ab-0e94-44ba-b3bd-12f7a06c623e", - "w": 18, - "x": 30, - "y": 33 - }, - "panelIndex": "a45187ab-0e94-44ba-b3bd-12f7a06c623e", - "title": "Vulnerability Severity", - "type": "lens" - } - ], - "timeRestore": false, - "title": "BBOT Dashboard", - "version": 1 - }, - "coreMigrationVersion": "8.8.0", - "created_at": "2024-03-21T19:29:20.744Z", - "id": "bbot-8abcb381-42b3-4d99-a177-c103255eedd9", - "managed": false, - "references": [ - { - "id": "logs-*", - "name": "ff18251e-b13b-42f6-8a10-6a6e61e2e74a:indexpattern-datasource-layer-b041b892-4b58-48f3-9f5e-52e0e604cfb0", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "ff18251e-b13b-42f6-8a10-6a6e61e2e74a:2604eb17-0109-4f38-993e-ed797031d791", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "e2b473cb-83a3-43b9-9845-01a865ebba81:indexpattern-datasource-layer-34706177-15e3-422e-942e-450494312e3f", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "e2b473cb-83a3-43b9-9845-01a865ebba81:d7a416f6-fbb4-4477-8760-363e18f9554c", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "8d154799-5342-4d9f-931a-8ac541b10235:indexpattern-datasource-layer-34706177-15e3-422e-942e-450494312e3f", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "8d154799-5342-4d9f-931a-8ac541b10235:ceb45dbd-8837-4fae-884c-5eef1f068cd9", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "fd6001b7-89f1-4008-b56e-9fee8d3111b1:indexpattern-datasource-layer-9236266e-4c6d-4cb0-8d5c-49493bf23532", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "fd6001b7-89f1-4008-b56e-9fee8d3111b1:52db8b89-498c-4aa2-ba42-d65b2025598f", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "ec50cd13-16ea-463b-8677-d6fc126fcaf8:indexpattern-datasource-layer-934f50cd-f1e9-47ea-be3a-3ceff354f1ad", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "ec50cd13-16ea-463b-8677-d6fc126fcaf8:1877a3bb-aa1f-420e-ad3b-b82ad23d1f0a", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "b7513787-adcc-4e88-8211-42e9c559f09c:indexpattern-datasource-layer-934f50cd-f1e9-47ea-be3a-3ceff354f1ad", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "b7513787-adcc-4e88-8211-42e9c559f09c:e6909ac9-f732-4420-a24d-69ffc4fe319c", - "type": "index-pattern" - }, - { - "id": "bbot-45ce1599-99e3-4c4e-9c1a-07254be0e274", - "name": "81963b3c-596f-4008-80de-286537f0c45d:panel_81963b3c-596f-4008-80de-286537f0c45d", - "type": "search" - }, - { - "id": "logs-*", - "name": "17ab65a3-eb4a-47df-8e8c-91c8ca504c67:indexpattern-datasource-layer-34706177-15e3-422e-942e-450494312e3f", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "17ab65a3-eb4a-47df-8e8c-91c8ca504c67:f6dc81d5-5b2a-40b4-b17a-2b8034ac3bb0", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "a45187ab-0e94-44ba-b3bd-12f7a06c623e:indexpattern-datasource-layer-34706177-15e3-422e-942e-450494312e3f", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "a45187ab-0e94-44ba-b3bd-12f7a06c623e:34e57322-6c1b-479e-95aa-318340186b2f", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "controlGroup_7b900e62-ba4a-468b-a99f-aa5bf4a3a526:optionsListDataView", - "type": "index-pattern" - } - ], - "type": "dashboard", - "typeMigrationVersion": "8.9.0" +{ + "attributes": { + "controlGroupInput": { + "chainingSystem": "HIERARCHICAL", + "controlStyle": "oneLine", + "ignoreParentSettingsJSON": "{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}", + "panelsJSON": "{\"7b900e62-ba4a-468b-a99f-aa5bf4a3a526\":{\"type\":\"optionsListControl\",\"order\":0,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"7b900e62-ba4a-468b-a99f-aa5bf4a3a526\",\"fieldName\":\"bbot.scan\",\"title\":\"Scan ID:\",\"grow\":true,\"width\":\"medium\",\"selectedOptions\":[],\"enhancements\":{}}}}" + }, + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-b041b892-4b58-48f3-9f5e-52e0e604cfb0", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "2604eb17-0109-4f38-993e-ed797031d791", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "b041b892-4b58-48f3-9f5e-52e0e604cfb0": { + "columnOrder": [ + "b77c2eee-54f7-4fa0-9aa6-936d9064ff4f", + "436a5f51-90a1-4193-b109-25b90ab29fb0" + ], + "columns": { + "436a5f51-90a1-4193-b109-25b90ab29fb0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Records", + "operationType": "count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "___records___" + }, + "b77c2eee-54f7-4fa0-9aa6-936d9064ff4f": { + "customLabel": true, + "dataType": "date", + "isBucketed": true, + "label": "Date of scan", + "operationType": "date_histogram", + "params": { + "dropPartials": false, + "includeEmptyRows": true, + "interval": "1w" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {} + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "2604eb17-0109-4f38-993e-ed797031d791", + "key": "event.dataset", + "negate": false, + "params": { + "query": "bbot.asm_intel" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "event.dataset": "bbot.asm_intel" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "436a5f51-90a1-4193-b109-25b90ab29fb0" + ], + "layerId": "b041b892-4b58-48f3-9f5e-52e0e604cfb0", + "layerType": "data", + "seriesType": "bar", + "xAccessor": "b77c2eee-54f7-4fa0-9aa6-936d9064ff4f", + "yConfig": [ + { + "color": "#e7664c", + "forAccessor": "436a5f51-90a1-4193-b109-25b90ab29fb0" + } + ] + } + ], + "legend": { + "isVisible": true, + "position": "right", + "showSingleSeries": false + }, + "preferredSeriesType": "bar", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "show" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false, + "timeRange": { + "from": "now-2y", + "to": "now" + } + }, + "gridData": { + "h": 6, + "i": "ff18251e-b13b-42f6-8a10-6a6e61e2e74a", + "w": 48, + "x": 0, + "y": 0 + }, + "panelIndex": "ff18251e-b13b-42f6-8a10-6a6e61e2e74a", + "title": "Scans over time", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "description": "This is a count of all url.domains found. There is some overlap between this field and the host.name field.", + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-34706177-15e3-422e-942e-450494312e3f", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "d7a416f6-fbb4-4477-8760-363e18f9554c", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "34706177-15e3-422e-942e-450494312e3f": { + "columnOrder": [ + "8847f861-0519-4914-b269-405389c0df68" + ], + "columns": { + "8847f861-0519-4914-b269-405389c0df68": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique Records of Domain", + "operationType": "unique_count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "url.domain" + } + }, + "incompleteColumns": {} + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "d7a416f6-fbb4-4477-8760-363e18f9554c", + "key": "event.dataset", + "negate": false, + "params": { + "query": "bbot.asm_intel" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "event.dataset": "bbot.asm_intel" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "color": "#E7664C", + "layerId": "34706177-15e3-422e-942e-450494312e3f", + "layerType": "data", + "metricAccessor": "8847f861-0519-4914-b269-405389c0df68" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "description": "This is a count of all url.domains found. There is some overlap between this field and the host.name field.", + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 5, + "i": "e2b473cb-83a3-43b9-9845-01a865ebba81", + "w": 15, + "x": 0, + "y": 6 + }, + "panelIndex": "e2b473cb-83a3-43b9-9845-01a865ebba81", + "title": "Unique Domains Found ", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "description": "This is a count of all related.hosts found. This field contains IPv4, IPv6 and Domain Names. ", + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-34706177-15e3-422e-942e-450494312e3f", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "ceb45dbd-8837-4fae-884c-5eef1f068cd9", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "34706177-15e3-422e-942e-450494312e3f": { + "columnOrder": [ + "8847f861-0519-4914-b269-405389c0df68" + ], + "columns": { + "8847f861-0519-4914-b269-405389c0df68": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Records Found for Related Hosts", + "operationType": "count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "related.hosts" + } + }, + "incompleteColumns": {} + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "ceb45dbd-8837-4fae-884c-5eef1f068cd9", + "key": "event.dataset", + "negate": false, + "params": { + "query": "bbot.asm_intel" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "event.dataset": "bbot.asm_intel" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "color": "#E7664C", + "layerId": "34706177-15e3-422e-942e-450494312e3f", + "layerType": "data", + "metricAccessor": "8847f861-0519-4914-b269-405389c0df68" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "description": "This is a count of all related.hosts found. This field contains IPv4, IPv6 and Domain Names. ", + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 5, + "i": "8d154799-5342-4d9f-931a-8ac541b10235", + "w": 15, + "x": 15, + "y": 6 + }, + "panelIndex": "8d154799-5342-4d9f-931a-8ac541b10235", + "title": "Related Hosts Found - Count of Records", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-9236266e-4c6d-4cb0-8d5c-49493bf23532", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "52db8b89-498c-4aa2-ba42-d65b2025598f", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "9236266e-4c6d-4cb0-8d5c-49493bf23532": { + "columnOrder": [ + "0896481f-8b3d-45f6-bb23-665ece65f846", + "8be8fd12-8e1b-45d8-93e5-3903ae887fc8" + ], + "columns": { + "0896481f-8b3d-45f6-bb23-665ece65f846": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Module", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "8be8fd12-8e1b-45d8-93e5-3903ae887fc8", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "bbot.module" + }, + "8be8fd12-8e1b-45d8-93e5-3903ae887fc8": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "___records___" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "52db8b89-498c-4aa2-ba42-d65b2025598f", + "key": "event.dataset", + "negate": false, + "params": { + "query": "bbot.asm_intel" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "event.dataset": "bbot.asm_intel" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "8be8fd12-8e1b-45d8-93e5-3903ae887fc8" + ], + "layerId": "9236266e-4c6d-4cb0-8d5c-49493bf23532", + "layerType": "data", + "position": "top", + "seriesType": "bar_horizontal", + "showGridlines": false, + "xAccessor": "0896481f-8b3d-45f6-bb23-665ece65f846", + "yConfig": [ + { + "color": "#e7664c", + "forAccessor": "8be8fd12-8e1b-45d8-93e5-3903ae887fc8" + } + ] + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar_horizontal", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 21, + "i": "fd6001b7-89f1-4008-b56e-9fee8d3111b1", + "w": 18, + "x": 30, + "y": 6 + }, + "panelIndex": "fd6001b7-89f1-4008-b56e-9fee8d3111b1", + "title": "Popular Module Findings", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-934f50cd-f1e9-47ea-be3a-3ceff354f1ad", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "1877a3bb-aa1f-420e-ad3b-b82ad23d1f0a", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "934f50cd-f1e9-47ea-be3a-3ceff354f1ad": { + "columnOrder": [ + "eebfb3a7-f1b5-4ca3-97e0-95eb896f8621", + "f4935493-86bc-4383-b231-651c7b375e59" + ], + "columns": { + "eebfb3a7-f1b5-4ca3-97e0-95eb896f8621": { + "dataType": "string", + "isBucketed": true, + "label": "Top 5 values of url.domain", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "f4935493-86bc-4383-b231-651c7b375e59", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "secondaryFields": [], + "size": 5 + }, + "scale": "ordinal", + "sourceField": "url.domain" + }, + "f4935493-86bc-4383-b231-651c7b375e59": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "___records___" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "1877a3bb-aa1f-420e-ad3b-b82ad23d1f0a", + "key": "event.dataset", + "negate": false, + "params": { + "query": "bbot.asm_intel" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "event.dataset": "bbot.asm_intel" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "categoryDisplay": "default", + "emptySizeRatio": 0.54, + "layerId": "934f50cd-f1e9-47ea-be3a-3ceff354f1ad", + "layerType": "data", + "legendDisplay": "default", + "metrics": [ + "f4935493-86bc-4383-b231-651c7b375e59" + ], + "nestedLegend": false, + "numberDisplay": "percent", + "primaryGroups": [ + "eebfb3a7-f1b5-4ca3-97e0-95eb896f8621" + ] + } + ], + "palette": { + "name": "negative", + "type": "palette" + }, + "shape": "donut" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsPie" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 16, + "i": "ec50cd13-16ea-463b-8677-d6fc126fcaf8", + "w": 15, + "x": 0, + "y": 11 + }, + "panelIndex": "ec50cd13-16ea-463b-8677-d6fc126fcaf8", + "title": "Top 5 Domain Records", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-934f50cd-f1e9-47ea-be3a-3ceff354f1ad", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "e6909ac9-f732-4420-a24d-69ffc4fe319c", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "934f50cd-f1e9-47ea-be3a-3ceff354f1ad": { + "columnOrder": [ + "eebfb3a7-f1b5-4ca3-97e0-95eb896f8621", + "f4935493-86bc-4383-b231-651c7b375e59" + ], + "columns": { + "eebfb3a7-f1b5-4ca3-97e0-95eb896f8621": { + "dataType": "string", + "isBucketed": true, + "label": "Top 5 values of related.hosts", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderBy": { + "columnId": "f4935493-86bc-4383-b231-651c7b375e59", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": false, + "parentFormat": { + "id": "terms" + }, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "related.hosts" + }, + "f4935493-86bc-4383-b231-651c7b375e59": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "___records___" + } + }, + "incompleteColumns": {}, + "sampling": 1 + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "e6909ac9-f732-4420-a24d-69ffc4fe319c", + "key": "event.dataset", + "negate": false, + "params": { + "query": "bbot.asm_intel" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "event.dataset": "bbot.asm_intel" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "layers": [ + { + "categoryDisplay": "default", + "emptySizeRatio": 0.54, + "layerId": "934f50cd-f1e9-47ea-be3a-3ceff354f1ad", + "layerType": "data", + "legendDisplay": "default", + "metrics": [ + "f4935493-86bc-4383-b231-651c7b375e59" + ], + "nestedLegend": false, + "numberDisplay": "percent", + "primaryGroups": [ + "eebfb3a7-f1b5-4ca3-97e0-95eb896f8621" + ] + } + ], + "palette": { + "name": "negative", + "type": "palette" + }, + "shape": "donut" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsPie" + }, + "description": "", + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 16, + "i": "b7513787-adcc-4e88-8211-42e9c559f09c", + "w": 15, + "x": 15, + "y": 11 + }, + "panelIndex": "b7513787-adcc-4e88-8211-42e9c559f09c", + "title": "Top 5 Related Hosts Found", + "type": "lens" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 21, + "i": "81963b3c-596f-4008-80de-286537f0c45d", + "w": 30, + "x": 0, + "y": 27 + }, + "panelIndex": "81963b3c-596f-4008-80de-286537f0c45d", + "panelRefName": "panel_81963b3c-596f-4008-80de-286537f0c45d", + "type": "search" + }, + { + "embeddableConfig": { + "attributes": { + "description": "", + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-34706177-15e3-422e-942e-450494312e3f", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "f6dc81d5-5b2a-40b4-b17a-2b8034ac3bb0", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "34706177-15e3-422e-942e-450494312e3f": { + "columnOrder": [ + "8847f861-0519-4914-b269-405389c0df68" + ], + "columns": { + "8847f861-0519-4914-b269-405389c0df68": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "vulnerability.severity : * " + }, + "isBucketed": false, + "label": "Hosts found with Vulnerabilities", + "operationType": "unique_count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "url.domain" + } + }, + "incompleteColumns": {} + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "f6dc81d5-5b2a-40b4-b17a-2b8034ac3bb0", + "key": "event.dataset", + "negate": false, + "params": { + "query": "bbot.asm_intel" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "event.dataset": "bbot.asm_intel" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "color": "#ffffff", + "layerId": "34706177-15e3-422e-942e-450494312e3f", + "layerType": "data", + "metricAccessor": "8847f861-0519-4914-b269-405389c0df68" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsMetric" + }, + "description": "", + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 6, + "i": "17ab65a3-eb4a-47df-8e8c-91c8ca504c67", + "w": 18, + "x": 30, + "y": 27 + }, + "panelIndex": "17ab65a3-eb4a-47df-8e8c-91c8ca504c67", + "title": "Vulnerable Hosts", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "logs-*", + "name": "indexpattern-datasource-layer-34706177-15e3-422e-942e-450494312e3f", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "34e57322-6c1b-479e-95aa-318340186b2f", + "type": "index-pattern" + } + ], + "state": { + "adHocDataViews": {}, + "datasourceStates": { + "formBased": { + "layers": { + "34706177-15e3-422e-942e-450494312e3f": { + "columnOrder": [ + "2170eae6-6ab4-4fce-ac60-fbbd4301da66", + "b6a09dd7-f423-43e6-8068-db01ebfa9855" + ], + "columns": { + "2170eae6-6ab4-4fce-ac60-fbbd4301da66": { + "dataType": "string", + "isBucketed": true, + "label": "Top 5 values of vulnerability.severity", + "operationType": "terms", + "params": { + "exclude": [], + "excludeIsRegex": false, + "include": [], + "includeIsRegex": false, + "missingBucket": false, + "orderAgg": { + "dataType": "number", + "isBucketed": false, + "label": "Count of records", + "operationType": "count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "___records___" + }, + "orderBy": { + "type": "custom" + }, + "orderDirection": "desc", + "otherBucket": true, + "parentFormat": { + "id": "terms" + }, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "vulnerability.severity" + }, + "b6a09dd7-f423-43e6-8068-db01ebfa9855": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Severity Percentage", + "operationType": "count", + "params": { + "emptyAsNull": true + }, + "scale": "ratio", + "sourceField": "vulnerability.severity" + } + }, + "incompleteColumns": {} + } + } + }, + "indexpattern": { + "layers": {} + }, + "textBased": { + "layers": {} + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "34e57322-6c1b-479e-95aa-318340186b2f", + "key": "event.dataset", + "negate": false, + "params": { + "query": "bbot.asm_intel" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "event.dataset": "bbot.asm_intel" + } + } + } + ], + "internalReferences": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "b6a09dd7-f423-43e6-8068-db01ebfa9855" + ], + "layerId": "34706177-15e3-422e-942e-450494312e3f", + "layerType": "data", + "palette": { + "name": "negative", + "type": "palette" + }, + "seriesType": "bar_percentage_stacked", + "splitAccessor": "2170eae6-6ab4-4fce-ac60-fbbd4301da66" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar_percentage_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "show" + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 15, + "i": "a45187ab-0e94-44ba-b3bd-12f7a06c623e", + "w": 18, + "x": 30, + "y": 33 + }, + "panelIndex": "a45187ab-0e94-44ba-b3bd-12f7a06c623e", + "title": "Vulnerability Severity", + "type": "lens" + } + ], + "timeRestore": false, + "title": "BBOT Dashboard", + "version": 1 + }, + "coreMigrationVersion": "8.8.0", + "created_at": "2024-03-21T19:29:20.744Z", + "id": "bbot-8abcb381-42b3-4d99-a177-c103255eedd9", + "managed": false, + "references": [ + { + "id": "logs-*", + "name": "ff18251e-b13b-42f6-8a10-6a6e61e2e74a:indexpattern-datasource-layer-b041b892-4b58-48f3-9f5e-52e0e604cfb0", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "ff18251e-b13b-42f6-8a10-6a6e61e2e74a:2604eb17-0109-4f38-993e-ed797031d791", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "e2b473cb-83a3-43b9-9845-01a865ebba81:indexpattern-datasource-layer-34706177-15e3-422e-942e-450494312e3f", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "e2b473cb-83a3-43b9-9845-01a865ebba81:d7a416f6-fbb4-4477-8760-363e18f9554c", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "8d154799-5342-4d9f-931a-8ac541b10235:indexpattern-datasource-layer-34706177-15e3-422e-942e-450494312e3f", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "8d154799-5342-4d9f-931a-8ac541b10235:ceb45dbd-8837-4fae-884c-5eef1f068cd9", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "fd6001b7-89f1-4008-b56e-9fee8d3111b1:indexpattern-datasource-layer-9236266e-4c6d-4cb0-8d5c-49493bf23532", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "fd6001b7-89f1-4008-b56e-9fee8d3111b1:52db8b89-498c-4aa2-ba42-d65b2025598f", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "ec50cd13-16ea-463b-8677-d6fc126fcaf8:indexpattern-datasource-layer-934f50cd-f1e9-47ea-be3a-3ceff354f1ad", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "ec50cd13-16ea-463b-8677-d6fc126fcaf8:1877a3bb-aa1f-420e-ad3b-b82ad23d1f0a", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "b7513787-adcc-4e88-8211-42e9c559f09c:indexpattern-datasource-layer-934f50cd-f1e9-47ea-be3a-3ceff354f1ad", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "b7513787-adcc-4e88-8211-42e9c559f09c:e6909ac9-f732-4420-a24d-69ffc4fe319c", + "type": "index-pattern" + }, + { + "id": "bbot-45ce1599-99e3-4c4e-9c1a-07254be0e274", + "name": "81963b3c-596f-4008-80de-286537f0c45d:panel_81963b3c-596f-4008-80de-286537f0c45d", + "type": "search" + }, + { + "id": "logs-*", + "name": "17ab65a3-eb4a-47df-8e8c-91c8ca504c67:indexpattern-datasource-layer-34706177-15e3-422e-942e-450494312e3f", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "17ab65a3-eb4a-47df-8e8c-91c8ca504c67:f6dc81d5-5b2a-40b4-b17a-2b8034ac3bb0", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "a45187ab-0e94-44ba-b3bd-12f7a06c623e:indexpattern-datasource-layer-34706177-15e3-422e-942e-450494312e3f", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "a45187ab-0e94-44ba-b3bd-12f7a06c623e:34e57322-6c1b-479e-95aa-318340186b2f", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "controlGroup_7b900e62-ba4a-468b-a99f-aa5bf4a3a526:optionsListDataView", + "type": "index-pattern" + } + ], + "type": "dashboard", + "typeMigrationVersion": "8.9.0" } \ No newline at end of file diff --git a/packages/bbot/kibana/search/bbot-45ce1599-99e3-4c4e-9c1a-07254be0e274.json b/packages/bbot/kibana/search/bbot-45ce1599-99e3-4c4e-9c1a-07254be0e274.json index 1298c05cb0a..81d2e0a0aa9 100644 --- a/packages/bbot/kibana/search/bbot-45ce1599-99e3-4c4e-9c1a-07254be0e274.json +++ b/packages/bbot/kibana/search/bbot-45ce1599-99e3-4c4e-9c1a-07254be0e274.json @@ -1,109 +1,109 @@ -{ - "attributes": { - "columns": [ - "url.domain", - "url.full", - "host.name", - "related.hosts", - "bbot.tags", - "bbot.module" - ], - "description": "This is used with the official BBOT dashboard.", - "grid": { - "columns": { - "@timestamp": { - "width": 303 - }, - "bbot.data.ASN.asn": { - "width": 268 - }, - "bbot.module": { - "width": 135 - }, - "bbot.tags": { - "width": 177 - }, - "host.ip": { - "width": 352 - }, - "host.name": { - "width": 201 - }, - "related.hosts": { - "width": 175 - }, - "url.domain": { - "width": 235 - }, - "url.full": { - "width": 350 - }, - "url.port": { - "width": 147 - } - } - }, - "hideChart": false, - "isTextBasedQuery": false, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [ - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", - "key": "event.dataset", - "negate": false, - "params": { - "query": "bbot.asm_intel" - }, - "type": "phrase" - }, - "query": { - "match_phrase": { - "event.dataset": "bbot.asm_intel" - } - } - } - ], - "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", - "query": { - "language": "kuery", - "query": "" - } - } - }, - "rowsPerPage": 50, - "sort": [ - [ - "@timestamp", - "desc" - ] - ], - "timeRestore": false, - "title": "[BBOT] Detailed Findings", - "usesAdHocDataView": false - }, - "coreMigrationVersion": "8.8.0", - "created_at": "2024-03-21T19:17:35.011Z", - "id": "bbot-45ce1599-99e3-4c4e-9c1a-07254be0e274", - "managed": false, - "references": [ - { - "id": "logs-*", - "name": "kibanaSavedObjectMeta.searchSourceJSON.index", - "type": "index-pattern" - }, - { - "id": "logs-*", - "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", - "type": "index-pattern" - } - ], - "type": "search", - "typeMigrationVersion": "8.0.0" +{ + "attributes": { + "columns": [ + "url.domain", + "url.full", + "host.name", + "related.hosts", + "bbot.tags", + "bbot.module" + ], + "description": "This is used with the official BBOT dashboard.", + "grid": { + "columns": { + "@timestamp": { + "width": 303 + }, + "bbot.data.ASN.asn": { + "width": 268 + }, + "bbot.module": { + "width": 135 + }, + "bbot.tags": { + "width": 177 + }, + "host.ip": { + "width": 352 + }, + "host.name": { + "width": 201 + }, + "related.hosts": { + "width": 175 + }, + "url.domain": { + "width": 235 + }, + "url.full": { + "width": 350 + }, + "url.port": { + "width": 147 + } + } + }, + "hideChart": false, + "isTextBasedQuery": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "event.dataset", + "negate": false, + "params": { + "query": "bbot.asm_intel" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "event.dataset": "bbot.asm_intel" + } + } + } + ], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "rowsPerPage": 50, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "timeRestore": false, + "title": "[BBOT] Detailed Findings", + "usesAdHocDataView": false + }, + "coreMigrationVersion": "8.8.0", + "created_at": "2024-03-21T19:17:35.011Z", + "id": "bbot-45ce1599-99e3-4c4e-9c1a-07254be0e274", + "managed": false, + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + } + ], + "type": "search", + "typeMigrationVersion": "8.0.0" } \ No newline at end of file diff --git a/packages/bbot/manifest.yml b/packages/bbot/manifest.yml index 6553f20da81..d66d347eb92 100644 --- a/packages/bbot/manifest.yml +++ b/packages/bbot/manifest.yml @@ -1,14 +1,14 @@ format_version: 3.1.2 name: bbot title: "BBOT (Bighuge BLS OSINT Tool)" -version: 0.1.0 +version: "0.2.0" description: "BBOT is a recursive internet scanner inspired by Spiderfoot, but designed to be faster, more reliable, and friendlier to pentesters, bug bounty hunters, and developers. " type: integration categories: - security conditions: kibana: - version: "^8.12.1" + version: "^8.13.0" elastic: subscription: "basic" screenshots: From 362d28215d616c1176fb81611ac005016380762b Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:16 +0930 Subject: [PATCH 016/121] [bitdefender] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/bitdefender --- packages/bitdefender/changelog.yml | 5 + .../push_configuration/fields/ecs.yml | 4 - .../push_notifications/fields/ecs.yml | 202 ------------------ .../push_statistics/fields/ecs.yml | 4 - packages/bitdefender/docs/README.md | 123 ----------- packages/bitdefender/manifest.yml | 4 +- 6 files changed, 7 insertions(+), 335 deletions(-) diff --git a/packages/bitdefender/changelog.yml b/packages/bitdefender/changelog.yml index 28fcd34360e..1211e0779e8 100644 --- a/packages/bitdefender/changelog.yml +++ b/packages/bitdefender/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.14.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.13.0" changes: - description: Update doc with input limitation collecting jsonRPC format. diff --git a/packages/bitdefender/data_stream/push_configuration/fields/ecs.yml b/packages/bitdefender/data_stream/push_configuration/fields/ecs.yml index e84a56b4f0e..c2fbf7e6e5a 100644 --- a/packages/bitdefender/data_stream/push_configuration/fields/ecs.yml +++ b/packages/bitdefender/data_stream/push_configuration/fields/ecs.yml @@ -6,7 +6,3 @@ name: data_stream.namespace - external: ecs name: '@timestamp' -- external: ecs - name: tags -- external: ecs - name: ecs.version diff --git a/packages/bitdefender/data_stream/push_notifications/fields/ecs.yml b/packages/bitdefender/data_stream/push_notifications/fields/ecs.yml index 944bfecf443..b7e148e95ed 100644 --- a/packages/bitdefender/data_stream/push_notifications/fields/ecs.yml +++ b/packages/bitdefender/data_stream/push_notifications/fields/ecs.yml @@ -6,205 +6,3 @@ name: data_stream.dataset - external: ecs name: data_stream.namespace -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: destination.user.domain -- external: ecs - name: destination.user.id -- external: ecs - name: destination.user.name -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.nat.ip -- external: ecs - name: ecs.version -- external: ecs - name: email.sender.address -- external: ecs - name: email.subject -- external: ecs - name: email.to.address -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.code -- external: ecs - name: event.created -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.outcome -- external: ecs - name: event.original -- external: ecs - name: event.provider -- external: ecs - name: event.sequence -- external: ecs - name: event.type -- external: ecs - name: event.id -- external: ecs - name: file.directory -- external: ecs - name: file.extension -- external: ecs - name: file.name -- external: ecs - name: file.path -- external: ecs - name: file.hash.md5 -- external: ecs - name: file.hash.sha256 -- external: ecs - name: file.size -- external: ecs - name: host.name -- external: ecs - name: log.level -- external: ecs - name: message -- external: ecs - name: network.type -- external: ecs - name: organization.id -- external: ecs - name: organization.name -- external: ecs - name: process.args -- external: ecs - name: process.args_count -- external: ecs - name: process.command_line -- external: ecs - name: process.entity_id -- external: ecs - name: process.executable -- external: ecs - name: process.name -- external: ecs - name: process.parent.executable -- external: ecs - name: process.parent.pid -- external: ecs - name: process.pid -- external: ecs - name: process.title -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: source.user.domain -- external: ecs - name: source.user.id -- external: ecs - name: source.user.name -- external: ecs - name: source.ip -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.name -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: tags -- external: ecs - name: user_agent.os.version -- external: ecs - name: threat.technique.id -- external: ecs - name: threat.technique.name -- external: ecs - name: threat.software.name -- external: ecs - name: user.domain -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.version -- external: ecs - name: user_agent.os.family -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.kernel -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.platform -- external: ecs - name: user_agent.os.type -- external: ecs - name: url.original -- external: ecs - name: url.domain -- external: ecs - name: url.extension -- external: ecs - name: url.path -- external: ecs - name: url.port -- external: ecs - name: url.registered_domain -- external: ecs - name: url.scheme -- external: ecs - name: url.subdomain -- external: ecs - name: url.top_level_domain -- external: ecs - name: url.query -- external: ecs - name: vulnerability.id diff --git a/packages/bitdefender/data_stream/push_statistics/fields/ecs.yml b/packages/bitdefender/data_stream/push_statistics/fields/ecs.yml index e84a56b4f0e..c2fbf7e6e5a 100644 --- a/packages/bitdefender/data_stream/push_statistics/fields/ecs.yml +++ b/packages/bitdefender/data_stream/push_statistics/fields/ecs.yml @@ -6,7 +6,3 @@ name: data_stream.namespace - external: ecs name: '@timestamp' -- external: ecs - name: tags -- external: ecs - name: ecs.version diff --git a/packages/bitdefender/docs/README.md b/packages/bitdefender/docs/README.md index c58cd32eff4..563c4f02c5b 100644 --- a/packages/bitdefender/docs/README.md +++ b/packages/bitdefender/docs/README.md @@ -306,17 +306,6 @@ All BitDefender GravityZone log events are available in the `bitdefender_gravity | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | | destination.nat.as.number | | long | | destination.nat.as.organization.name | | keyword | | destination.nat.geo.city_name | | keyword | @@ -326,115 +315,7 @@ All BitDefender GravityZone log events are available in the `bitdefender_gravity | destination.nat.geo.location | | geo_point | | destination.nat.geo.region_iso_code | | keyword | | destination.nat.geo.region_name | | keyword | -| destination.nat.ip | Translated ip of destination based NAT sessions (e.g. internet to private DMZ) Typically used with load balancers, firewalls, or routers. | ip | -| destination.port | Port of the destination. | long | -| destination.user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| destination.user.id | Unique identifier of the user. | keyword | -| destination.user.name | Short name or login of the user. | keyword | -| destination.user.name.text | Multi-field of `destination.user.name`. | match_only_text | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.sender.address | Per RFC 5322, specifies the address responsible for the actual transmission of the message. | keyword | -| email.subject | A brief summary of the topic of the message. | keyword | -| email.subject.text | Multi-field of `email.subject`. | match_only_text | -| email.to.address | The email address of recipient | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.sequence | Sequence number of the event. The sequence number is a value published by some event sources, to make the exact ordering of events unambiguous, regardless of the timestamp precision. | long | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.directory | Directory where the file is located. It should include the drive letter, when appropriate. | keyword | -| file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| file.hash.md5 | MD5 hash. | keyword | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | input.type | | keyword | -| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | -| organization.id | Unique identifier for the organization. | keyword | -| organization.name | Organization name. | keyword | -| organization.name.text | Multi-field of `organization.name`. | match_only_text | -| process.args | Array of process arguments, starting with the absolute path to the executable. May be filtered to protect sensitive information. | keyword | -| process.args_count | Length of the process.args array. This field can be useful for querying or performing bucket analysis on how many arguments were provided to start a process. More arguments may be an indication of suspicious activity. | long | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.executable | Absolute path to the process executable. | keyword | -| process.executable.text | Multi-field of `process.executable`. | match_only_text | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.parent.executable | Absolute path to the process executable. | keyword | -| process.parent.executable.text | Multi-field of `process.parent.executable`. | match_only_text | -| process.parent.pid | Process id. | long | -| process.pid | Process id. | long | -| process.title | Process title. The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. | keyword | -| process.title.text | Multi-field of `process.title`. | match_only_text | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| source.user.id | Unique identifier of the user. | keyword | -| source.user.name | Short name or login of the user. | keyword | -| source.user.name.text | Multi-field of `source.user.name`. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| threat.software.name | The name of the software used by this threat to conduct behavior commonly modeled using MITRE ATT&CK®. While not required, you can use a MITRE ATT&CK® software name. | keyword | -| threat.technique.id | The id of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.name | The name of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.name.text | Multi-field of `threat.technique.name`. | match_only_text | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.kernel | Operating system kernel version as a raw string. | keyword | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| user_agent.os.type | Use the `os.type` field to categorize the operating system into one of the broad commercial families. If the OS you're dealing with is not listed as an expected value, the field should not be populated. Please let us know by opening an issue with ECS, to propose its addition. | keyword | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | -| vulnerability.id | The identification (ID) is the number portion of a vulnerability entry. It includes a unique identification number for the vulnerability. For example (https://cve.mitre.org/about/faqs.html#what_is_cve_id)[Common Vulnerabilities and Exposure CVE ID] | keyword | An example event for `push_notifications` looks as following: @@ -556,9 +437,7 @@ All BitDefender GravityZone push notification configuration states are available | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | input.type | | keyword | -| tags | List of keywords used to tag each event. | keyword | An example event for `push_configuration` looks as following: @@ -676,9 +555,7 @@ All BitDefender GravityZone push notification statistics are available in the `b | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | input.type | | keyword | -| tags | List of keywords used to tag each event. | keyword | An example event for `push_statistics` looks as following: diff --git a/packages/bitdefender/manifest.yml b/packages/bitdefender/manifest.yml index 7436557c080..e60834c450f 100644 --- a/packages/bitdefender/manifest.yml +++ b/packages/bitdefender/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: bitdefender title: "BitDefender" -version: "1.13.0" +version: "1.14.0" source: license: "Elastic-2.0" description: "Ingest BitDefender GravityZone logs and data" @@ -10,7 +10,7 @@ categories: - security conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" elastic: subscription: "basic" screenshots: From 551089c7987991069ecece18d51c4b6f9a21f25f Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:19 +0930 Subject: [PATCH 017/121] [bitwarden] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/bitwarden --- packages/bitwarden/_dev/build/build.yml | 1 - packages/bitwarden/changelog.yml | 5 +++++ packages/bitwarden/data_stream/collection/fields/beats.yml | 3 --- packages/bitwarden/data_stream/event/fields/beats.yml | 3 --- packages/bitwarden/data_stream/group/fields/beats.yml | 3 --- packages/bitwarden/data_stream/member/fields/beats.yml | 3 --- packages/bitwarden/data_stream/policy/fields/beats.yml | 3 --- packages/bitwarden/docs/README.md | 5 ----- packages/bitwarden/manifest.yml | 4 ++-- 9 files changed, 7 insertions(+), 23 deletions(-) diff --git a/packages/bitwarden/_dev/build/build.yml b/packages/bitwarden/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/bitwarden/_dev/build/build.yml +++ b/packages/bitwarden/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/bitwarden/changelog.yml b/packages/bitwarden/changelog.yml index 498ef88c757..5aeeeef8af1 100644 --- a/packages/bitwarden/changelog.yml +++ b/packages/bitwarden/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.13.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.12.0" changes: - description: Improve handling of empty responses. diff --git a/packages/bitwarden/data_stream/collection/fields/beats.yml b/packages/bitwarden/data_stream/collection/fields/beats.yml index 3415608ae37..cc9fcebf29b 100644 --- a/packages/bitwarden/data_stream/collection/fields/beats.yml +++ b/packages/bitwarden/data_stream/collection/fields/beats.yml @@ -1,9 +1,6 @@ - name: input.type description: Type of Filebeat input. type: keyword -- name: tags - type: keyword - description: User defined tags. - name: log.offset type: long description: Log offset. diff --git a/packages/bitwarden/data_stream/event/fields/beats.yml b/packages/bitwarden/data_stream/event/fields/beats.yml index 3415608ae37..cc9fcebf29b 100644 --- a/packages/bitwarden/data_stream/event/fields/beats.yml +++ b/packages/bitwarden/data_stream/event/fields/beats.yml @@ -1,9 +1,6 @@ - name: input.type description: Type of Filebeat input. type: keyword -- name: tags - type: keyword - description: User defined tags. - name: log.offset type: long description: Log offset. diff --git a/packages/bitwarden/data_stream/group/fields/beats.yml b/packages/bitwarden/data_stream/group/fields/beats.yml index 3415608ae37..cc9fcebf29b 100644 --- a/packages/bitwarden/data_stream/group/fields/beats.yml +++ b/packages/bitwarden/data_stream/group/fields/beats.yml @@ -1,9 +1,6 @@ - name: input.type description: Type of Filebeat input. type: keyword -- name: tags - type: keyword - description: User defined tags. - name: log.offset type: long description: Log offset. diff --git a/packages/bitwarden/data_stream/member/fields/beats.yml b/packages/bitwarden/data_stream/member/fields/beats.yml index 3415608ae37..cc9fcebf29b 100644 --- a/packages/bitwarden/data_stream/member/fields/beats.yml +++ b/packages/bitwarden/data_stream/member/fields/beats.yml @@ -1,9 +1,6 @@ - name: input.type description: Type of Filebeat input. type: keyword -- name: tags - type: keyword - description: User defined tags. - name: log.offset type: long description: Log offset. diff --git a/packages/bitwarden/data_stream/policy/fields/beats.yml b/packages/bitwarden/data_stream/policy/fields/beats.yml index 3415608ae37..cc9fcebf29b 100644 --- a/packages/bitwarden/data_stream/policy/fields/beats.yml +++ b/packages/bitwarden/data_stream/policy/fields/beats.yml @@ -1,9 +1,6 @@ - name: input.type description: Type of Filebeat input. type: keyword -- name: tags - type: keyword - description: User defined tags. - name: log.offset type: long description: Log offset. diff --git a/packages/bitwarden/docs/README.md b/packages/bitwarden/docs/README.md index b9b0a66254d..3b5fa2d748b 100644 --- a/packages/bitwarden/docs/README.md +++ b/packages/bitwarden/docs/README.md @@ -122,7 +122,6 @@ An example event for `collection` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Event @@ -266,7 +265,6 @@ An example event for `event` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Group @@ -367,7 +365,6 @@ An example event for `group` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Member @@ -493,7 +490,6 @@ An example event for `member` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Policy @@ -615,4 +611,3 @@ An example event for `policy` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | diff --git a/packages/bitwarden/manifest.yml b/packages/bitwarden/manifest.yml index e709799da03..c5b43233a39 100644 --- a/packages/bitwarden/manifest.yml +++ b/packages/bitwarden/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: bitwarden title: Bitwarden -version: "1.12.0" +version: "1.13.0" source: license: Elastic-2.0 description: Collect logs from Bitwarden with Elastic Agent. @@ -11,7 +11,7 @@ categories: - credential_management conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" elastic: subscription: "basic" screenshots: From 7e50996ab9b881ed25875c878f302d9656efe1db Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:21 +0930 Subject: [PATCH 018/121] [box_events] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/box_events --- packages/box_events/changelog.yml | 5 + .../data_stream/events/fields/agent.yml | 57 ------- .../data_stream/events/fields/ecs.yml | 140 ------------------ packages/box_events/docs/README.md | 102 ------------- packages/box_events/manifest.yml | 4 +- 5 files changed, 7 insertions(+), 301 deletions(-) delete mode 100644 packages/box_events/data_stream/events/fields/ecs.yml diff --git a/packages/box_events/changelog.yml b/packages/box_events/changelog.yml index 381b0b953ad..f5b2ae08ebe 100644 --- a/packages/box_events/changelog.yml +++ b/packages/box_events/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.9.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.8.0" changes: - description: Use `event_id` field for document fingerprinting. diff --git a/packages/box_events/data_stream/events/fields/agent.yml b/packages/box_events/data_stream/events/fields/agent.yml index 6048dca3d44..8e1c9f999da 100644 --- a/packages/box_events/data_stream/events/fields/agent.yml +++ b/packages/box_events/data_stream/events/fields/agent.yml @@ -1,65 +1,12 @@ - name: cloud type: group fields: - - name: account.id - external: ecs - - name: availability_zone - external: ecs - - name: instance.id - external: ecs - - name: instance.name - external: ecs - - name: machine.type - external: ecs - - name: provider - external: ecs - - name: region - external: ecs - - name: project.id - external: ecs - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - type: group - fields: - - name: id - external: ecs - - name: image.name - external: ecs - - name: labels - external: ecs - - name: name - external: ecs - name: host type: group fields: - - name: architecture - external: ecs - - name: domain - external: ecs - - name: hostname - external: ecs - - name: id - external: ecs - - name: ip - external: ecs - - name: mac - external: ecs - - name: name - external: ecs - - name: os.family - external: ecs - - name: os.kernel - external: ecs - - name: os.name - external: ecs - - name: os.platform - external: ecs - - name: os.version - external: ecs - - name: type - external: ecs - name: containerized type: boolean description: > @@ -82,10 +29,6 @@ description: > Percent CPU used. This value is normalized by the number of CPU cores and it ranges from 0 to 1. - - name: disk.read.bytes - external: ecs - - name: disk.write.bytes - external: ecs - name: network.in.bytes type: long description: > diff --git a/packages/box_events/data_stream/events/fields/ecs.yml b/packages/box_events/data_stream/events/fields/ecs.yml deleted file mode 100644 index f0aaa28f258..00000000000 --- a/packages/box_events/data_stream/events/fields/ecs.yml +++ /dev/null @@ -1,140 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: client.ip -- external: ecs - name: client.user.id -- external: ecs - name: client.user.full_name -- external: ecs - name: client.user.email -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.risk_score -- external: ecs - name: event.sequence -- external: ecs - name: event.type -- external: ecs - name: file.type -- external: ecs - name: file.directory -- external: ecs - name: file.name -- external: ecs - name: file.created -- external: ecs - name: file.ctime -- external: ecs - name: file.mtime -- external: ecs - name: file.size -- external: ecs - name: file.hash.sha1 -- external: ecs - name: message -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: rule.category -- external: ecs - name: rule.id -- external: ecs - name: rule.uuid -- external: ecs - name: rule.name -- external: ecs - name: tags -- external: ecs - name: threat.indicator.as.number -- external: ecs - name: threat.indicator.as.organization.name -- external: ecs - name: threat.indicator.confidence -- external: ecs - name: threat.indicator.description -- external: ecs - name: threat.enrichments -- external: ecs - name: threat.enrichments.indicator.as.number -- external: ecs - name: threat.enrichments.indicator.as.organization.name -- external: ecs - name: threat.enrichments.indicator.description -- external: ecs - name: threat.enrichments.indicator.first_seen -- external: ecs - name: threat.enrichments.indicator.geo.city_name -- external: ecs - name: threat.enrichments.indicator.geo.continent_name -- external: ecs - name: threat.enrichments.indicator.geo.country_iso_code -- external: ecs - name: threat.enrichments.indicator.geo.country_name -- external: ecs - name: threat.enrichments.indicator.geo.location -- external: ecs - name: threat.enrichments.indicator.geo.region_iso_code -- external: ecs - name: threat.enrichments.indicator.geo.region_name -- external: ecs - name: threat.enrichments.indicator.ip -- external: ecs - name: threat.enrichments.indicator.last_seen -- external: ecs - name: threat.enrichments.indicator.provider -- external: ecs - name: threat.enrichments.indicator.reference -- external: ecs - name: threat.enrichments.indicator.type -- external: ecs - name: threat.indicator.first_seen -- external: ecs - name: threat.indicator.geo.city_name -- external: ecs - name: threat.indicator.geo.continent_name -- external: ecs - name: threat.indicator.geo.country_iso_code -- external: ecs - name: threat.indicator.geo.country_name -- name: threat.indicator.geo.location - external: ecs -- external: ecs - name: threat.indicator.geo.region_iso_code -- external: ecs - name: threat.indicator.geo.region_name -- external: ecs - name: threat.indicator.ip -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.reference -- external: ecs - name: threat.indicator.sightings -- external: ecs - name: threat.indicator.type -- external: ecs - name: user.effective.id -- external: ecs - name: user.effective.name -- external: ecs - name: user.effective.email diff --git a/packages/box_events/docs/README.md b/packages/box_events/docs/README.md index 320dcfb9268..b24efb95035 100644 --- a/packages/box_events/docs/README.md +++ b/packages/box_events/docs/README.md @@ -252,124 +252,22 @@ Preserves a raw copy of the original event, added to the field `event.original`. | box.source.synced | Legacy property for compatibility with Box Desktop | boolean | | box.source.timezone | Timezone | boolean | | box.source.trashed_at | The time at which this file was put in the trash | boolean | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| client.user.email | User email address. | keyword | -| client.user.full_name | User's full name, if available. | keyword | -| client.user.full_name.text | Multi-field of `client.user.full_name`. | match_only_text | -| client.user.id | Unique identifier of the user. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | -| event.risk_score | Risk score or priority of the event (e.g. security solutions). Use your system's original value here. | float | -| event.sequence | Sequence number of the event. The sequence number is a value published by some event sources, to make the exact ordering of events unambiguous, regardless of the timestamp precision. | long | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.created | File creation time. Note that not all filesystems store the creation time. | date | -| file.ctime | Last time the file attributes or metadata changed. Note that changes to the file content will update `mtime`. This implies `ctime` will be adjusted at the same time, since `mtime` is an attribute of the file. | date | -| file.directory | Directory where the file is located. It should include the drive letter, when appropriate. | keyword | -| file.hash.sha1 | SHA1 hash. | keyword | -| file.mtime | Last time the file content was modified. | date | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| file.type | File type (file, dir, or symlink). | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.cpu.pct | Percent CPU used. This value is normalized by the number of CPU cores and it ranges from 0 to 1. | scaled_float | -| host.disk.read.bytes | The total number of bytes (gauge) read successfully (aggregated from all disks) since the last metric collection. | long | -| host.disk.write.bytes | The total number of bytes (gauge) written successfully (aggregated from all disks) since the last metric collection. | long | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.network.in.bytes | The number of bytes received on all network interfaces by the host in a given period of time. | long | | host.network.in.packets | The number of packets received on all network interfaces by the host in a given period of time. | long | | host.network.out.bytes | The number of bytes sent out on all network interfaces by the host in a given period of time. | long | | host.network.out.packets | The number of packets sent out on all network interfaces by the host in a given period of time. | long | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | related.description | Array of `description` derived from `threat[.enrichments].indicator.description` | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | | related.indicator_type | Array of `indicator_type` derived from `threat[.enrichments].indicator.type` | keyword | -| related.ip | All of the IPs seen on your event. | ip | | related.location | Array of `location` derived from `related.ip` | geo_point | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.category | A categorization value keyword used by the entity using the rule for detection of this event. | keyword | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| rule.uuid | A rule ID that is unique within the scope of a set or group of agents, observers, or other entities using the rule for detection of this event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| threat.enrichments | A list of associated indicators objects enriching the event, and the context of that association/enrichment. | nested | -| threat.enrichments.indicator.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| threat.enrichments.indicator.as.organization.name | Organization name. | keyword | -| threat.enrichments.indicator.as.organization.name.text | Multi-field of `threat.enrichments.indicator.as.organization.name`. | match_only_text | -| threat.enrichments.indicator.description | Describes the type of action conducted by the threat. | keyword | -| threat.enrichments.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.enrichments.indicator.geo.city_name | City name. | keyword | -| threat.enrichments.indicator.geo.continent_name | Name of the continent. | keyword | -| threat.enrichments.indicator.geo.country_iso_code | Country ISO code. | keyword | -| threat.enrichments.indicator.geo.country_name | Country name. | keyword | -| threat.enrichments.indicator.geo.location | Longitude and latitude. | geo_point | -| threat.enrichments.indicator.geo.region_iso_code | Region ISO code. | keyword | -| threat.enrichments.indicator.geo.region_name | Region name. | keyword | -| threat.enrichments.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | -| threat.enrichments.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.enrichments.indicator.provider | The name of the indicator's provider. | keyword | -| threat.enrichments.indicator.reference | Reference URL linking to additional information about this indicator. | keyword | -| threat.enrichments.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| threat.indicator.as.organization.name | Organization name. | keyword | -| threat.indicator.as.organization.name.text | Multi-field of `threat.indicator.as.organization.name`. | match_only_text | -| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword | -| threat.indicator.description | Describes the type of action conducted by the threat. | keyword | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.geo.city_name | City name. | keyword | -| threat.indicator.geo.continent_name | Name of the continent. | keyword | -| threat.indicator.geo.country_iso_code | Country ISO code. | keyword | -| threat.indicator.geo.country_name | Country name. | keyword | -| threat.indicator.geo.location | Longitude and latitude. | geo_point | -| threat.indicator.geo.region_iso_code | Region ISO code. | keyword | -| threat.indicator.geo.region_name | Region name. | keyword | -| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.reference | Reference URL linking to additional information about this indicator. | keyword | -| threat.indicator.sightings | Number of times this indicator was observed conducting threat activity. | long | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| user.effective.email | User email address. | keyword | -| user.effective.id | Unique identifier of the user. | keyword | -| user.effective.name | Short name or login of the user. | keyword | -| user.effective.name.text | Multi-field of `user.effective.name`. | match_only_text | diff --git a/packages/box_events/manifest.yml b/packages/box_events/manifest.yml index f6f41d49534..b582b23eba2 100644 --- a/packages/box_events/manifest.yml +++ b/packages/box_events/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: box_events title: Box Events -version: "2.8.0" +version: "2.9.0" description: "Collect logs from Box with Elastic Agent" type: integration categories: @@ -9,7 +9,7 @@ categories: - productivity_security conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" screenshots: - src: /img/box_screenshot.png title: "[Logs Box Events Integration] Events Dashboard" From a9b5a9c70353d59469f4881adaef51778f9b5fdf Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:26 +0930 Subject: [PATCH 019/121] [carbon_black_cloud] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/carbon_black_cloud --- packages/carbon_black_cloud/changelog.yml | 5 + .../data_stream/alert/fields/agent.yml | 134 +-------- .../data_stream/alert/fields/ecs.yml | 36 --- .../data_stream/alert_v7/fields/agent.yml | 134 +-------- .../data_stream/alert_v7/fields/ecs.yml | 58 ---- .../fields/agent.yml | 134 +-------- .../fields/ecs.yml | 14 - .../data_stream/audit/fields/agent.yml | 134 +-------- .../data_stream/audit/fields/ecs.yml | 24 -- .../endpoint_event/fields/agent.yml | 134 +-------- .../data_stream/endpoint_event/fields/ecs.yml | 80 ----- .../watchlist_hit/fields/agent.yml | 134 +-------- .../data_stream/watchlist_hit/fields/ecs.yml | 46 --- packages/carbon_black_cloud/docs/README.md | 276 ------------------ packages/carbon_black_cloud/manifest.yml | 4 +- 15 files changed, 13 insertions(+), 1334 deletions(-) delete mode 100644 packages/carbon_black_cloud/data_stream/alert/fields/ecs.yml delete mode 100644 packages/carbon_black_cloud/data_stream/alert_v7/fields/ecs.yml delete mode 100644 packages/carbon_black_cloud/data_stream/asset_vulnerability_summary/fields/ecs.yml delete mode 100644 packages/carbon_black_cloud/data_stream/audit/fields/ecs.yml delete mode 100644 packages/carbon_black_cloud/data_stream/endpoint_event/fields/ecs.yml delete mode 100644 packages/carbon_black_cloud/data_stream/watchlist_hit/fields/ecs.yml diff --git a/packages/carbon_black_cloud/changelog.yml b/packages/carbon_black_cloud/changelog.yml index 107d90505d5..acede149602 100644 --- a/packages/carbon_black_cloud/changelog.yml +++ b/packages/carbon_black_cloud/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.2.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.1.0" changes: - description: Improve handling of empty responses. diff --git a/packages/carbon_black_cloud/data_stream/alert/fields/agent.yml b/packages/carbon_black_cloud/data_stream/alert/fields/agent.yml index bf2dfff6756..48f513b61aa 100644 --- a/packages/carbon_black_cloud/data_stream/alert/fields/agent.yml +++ b/packages/carbon_black_cloud/data_stream/alert/fields/agent.yml @@ -5,147 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/carbon_black_cloud/data_stream/alert/fields/ecs.yml b/packages/carbon_black_cloud/data_stream/alert/fields/ecs.yml deleted file mode 100644 index 1a0c0a5368f..00000000000 --- a/packages/carbon_black_cloud/data_stream/alert/fields/ecs.yml +++ /dev/null @@ -1,36 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.created -- external: ecs - name: event.end -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.reason -- external: ecs - name: event.severity -- external: ecs - name: event.start -- external: ecs - name: process.entity_id -- external: ecs - name: process.executable -- external: ecs - name: process.name -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.user -- external: ecs - name: tags -- external: ecs - name: user.domain -- external: ecs - name: user.name diff --git a/packages/carbon_black_cloud/data_stream/alert_v7/fields/agent.yml b/packages/carbon_black_cloud/data_stream/alert_v7/fields/agent.yml index bf2dfff6756..48f513b61aa 100644 --- a/packages/carbon_black_cloud/data_stream/alert_v7/fields/agent.yml +++ b/packages/carbon_black_cloud/data_stream/alert_v7/fields/agent.yml @@ -5,147 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/carbon_black_cloud/data_stream/alert_v7/fields/ecs.yml b/packages/carbon_black_cloud/data_stream/alert_v7/fields/ecs.yml deleted file mode 100644 index 397d0af5c9e..00000000000 --- a/packages/carbon_black_cloud/data_stream/alert_v7/fields/ecs.yml +++ /dev/null @@ -1,58 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.created -- external: ecs - name: event.end -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.reason -- external: ecs - name: event.severity -- external: ecs - name: event.start -- external: ecs - name: process.entity_id -- external: ecs - name: process.executable -- external: ecs - name: process.name -- external: ecs - name: process.command_line -- external: ecs - name: process.pid -- external: ecs - name: process.hash.sha256 -- external: ecs - name: process.hash.md5 -- external: ecs - name: process.parent.entity_id -- external: ecs - name: process.parent.executable -- external: ecs - name: process.parent.name -- external: ecs - name: process.parent.command_line -- external: ecs - name: process.parent.pid -- external: ecs - name: process.parent.hash.sha256 -- external: ecs - name: process.parent.hash.md5 -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.user -- external: ecs - name: tags -- external: ecs - name: user.domain -- external: ecs - name: user.name diff --git a/packages/carbon_black_cloud/data_stream/asset_vulnerability_summary/fields/agent.yml b/packages/carbon_black_cloud/data_stream/asset_vulnerability_summary/fields/agent.yml index bf2dfff6756..48f513b61aa 100644 --- a/packages/carbon_black_cloud/data_stream/asset_vulnerability_summary/fields/agent.yml +++ b/packages/carbon_black_cloud/data_stream/asset_vulnerability_summary/fields/agent.yml @@ -5,147 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/carbon_black_cloud/data_stream/asset_vulnerability_summary/fields/ecs.yml b/packages/carbon_black_cloud/data_stream/asset_vulnerability_summary/fields/ecs.yml deleted file mode 100644 index e40d2676a61..00000000000 --- a/packages/carbon_black_cloud/data_stream/asset_vulnerability_summary/fields/ecs.yml +++ /dev/null @@ -1,14 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.created -- external: ecs - name: event.original -- external: ecs - name: related.hosts -- external: ecs - name: tags -- external: ecs - name: vulnerability.score.base -- external: ecs - name: vulnerability.severity diff --git a/packages/carbon_black_cloud/data_stream/audit/fields/agent.yml b/packages/carbon_black_cloud/data_stream/audit/fields/agent.yml index bf2dfff6756..48f513b61aa 100644 --- a/packages/carbon_black_cloud/data_stream/audit/fields/agent.yml +++ b/packages/carbon_black_cloud/data_stream/audit/fields/agent.yml @@ -5,147 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/carbon_black_cloud/data_stream/audit/fields/ecs.yml b/packages/carbon_black_cloud/data_stream/audit/fields/ecs.yml deleted file mode 100644 index 3edf889adfa..00000000000 --- a/packages/carbon_black_cloud/data_stream/audit/fields/ecs.yml +++ /dev/null @@ -1,24 +0,0 @@ -- external: ecs - name: client.ip -- external: ecs - name: client.user.id -- external: ecs - name: ecs.version -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.reason -- external: ecs - name: organization.name -- external: ecs - name: related.ip -- external: ecs - name: tags -- external: ecs - name: url.original diff --git a/packages/carbon_black_cloud/data_stream/endpoint_event/fields/agent.yml b/packages/carbon_black_cloud/data_stream/endpoint_event/fields/agent.yml index bf2dfff6756..48f513b61aa 100644 --- a/packages/carbon_black_cloud/data_stream/endpoint_event/fields/agent.yml +++ b/packages/carbon_black_cloud/data_stream/endpoint_event/fields/agent.yml @@ -5,147 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/carbon_black_cloud/data_stream/endpoint_event/fields/ecs.yml b/packages/carbon_black_cloud/data_stream/endpoint_event/fields/ecs.yml deleted file mode 100644 index fa3c12ea3f5..00000000000 --- a/packages/carbon_black_cloud/data_stream/endpoint_event/fields/ecs.yml +++ /dev/null @@ -1,80 +0,0 @@ -- external: ecs - name: client.ip -- external: ecs - name: client.port -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: dll.hash.md5 -- external: ecs - name: dll.hash.sha256 -- external: ecs - name: dll.path -- external: ecs - name: ecs.version -- external: ecs - name: event.action -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.original -- external: ecs - name: event.reason -- external: ecs - name: file.hash.md5 -- external: ecs - name: file.hash.sha256 -- external: ecs - name: file.path -- external: ecs - name: network.direction -- external: ecs - name: network.transport -- external: ecs - name: process.command_line -- external: ecs - name: process.entity_id -- external: ecs - name: process.executable -- external: ecs - name: process.hash.md5 -- external: ecs - name: process.hash.sha256 -- external: ecs - name: process.parent.command_line -- external: ecs - name: process.parent.entity_id -- external: ecs - name: process.parent.executable -- external: ecs - name: process.parent.hash.md5 -- external: ecs - name: process.parent.hash.sha256 -- external: ecs - name: process.parent.pid -- external: ecs - name: process.pid -- external: ecs - name: registry.path -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: source.address -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: tags -- external: ecs - name: user.domain diff --git a/packages/carbon_black_cloud/data_stream/watchlist_hit/fields/agent.yml b/packages/carbon_black_cloud/data_stream/watchlist_hit/fields/agent.yml index bf2dfff6756..48f513b61aa 100644 --- a/packages/carbon_black_cloud/data_stream/watchlist_hit/fields/agent.yml +++ b/packages/carbon_black_cloud/data_stream/watchlist_hit/fields/agent.yml @@ -5,147 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/carbon_black_cloud/data_stream/watchlist_hit/fields/ecs.yml b/packages/carbon_black_cloud/data_stream/watchlist_hit/fields/ecs.yml deleted file mode 100644 index 484cce18ef3..00000000000 --- a/packages/carbon_black_cloud/data_stream/watchlist_hit/fields/ecs.yml +++ /dev/null @@ -1,46 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.severity -- external: ecs - name: process.command_line -- external: ecs - name: process.entity_id -- external: ecs - name: process.executable -- external: ecs - name: process.hash.md5 -- external: ecs - name: process.hash.sha256 -- external: ecs - name: process.parent.command_line -- external: ecs - name: process.parent.entity_id -- external: ecs - name: process.parent.executable -- external: ecs - name: process.parent.hash.md5 -- external: ecs - name: process.parent.hash.sha256 -- external: ecs - name: process.parent.pid -- external: ecs - name: process.pid -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: tags -- external: ecs - name: user.domain diff --git a/packages/carbon_black_cloud/docs/README.md b/packages/carbon_black_cloud/docs/README.md index f7f0a558d3b..409c00a0729 100644 --- a/packages/carbon_black_cloud/docs/README.md +++ b/packages/carbon_black_cloud/docs/README.md @@ -159,52 +159,17 @@ An example event for `audit` looks as following: | @timestamp | Event timestamp. | date | | carbon_black_cloud.audit.flagged | true if action is failed otherwise false. | boolean | | carbon_black_cloud.audit.verbose | true if verbose audit log otherwise false. | boolean | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| client.user.id | Unique identifier of the user. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.reason | Reason why this event happened, according to the source. This describes the why of a particular action or outcome captured in the event. Where `event.action` captures the action from the event, `event.reason` describes why that action was taken. For example, a web proxy with an `event.action` which denied the request may also populate `event.reason` with the reason why (e.g. `blocked site`). | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.mac | Host mac addresses. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| organization.name | Organization name. | keyword | -| organization.name.text | Multi-field of `organization.name`. | match_only_text | -| related.ip | All of the IPs seen on your event. | ip | -| tags | List of keywords used to tag each event. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | ### Alert @@ -380,59 +345,17 @@ An example event for `alert` looks as following: | carbon_black_cloud.alert.workflow.last_update_time | The last update time of workflow. | date | | carbon_black_cloud.alert.workflow.remediation | N/A. | keyword | | carbon_black_cloud.alert.workflow.state | The state of workflow. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.reason | Reason why this event happened, according to the source. This describes the why of a particular action or outcome captured in the event. Where `event.action` captures the action from the event, `event.reason` describes why that action was taken. For example, a web proxy with an `event.action` which denied the request may also populate `event.reason` with the reason why (e.g. `blocked site`). | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.mac | Host mac addresses. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| process.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.executable | Absolute path to the process executable. | keyword | -| process.executable.text | Multi-field of `process.executable`. | match_only_text | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | ### Alert @@ -783,74 +706,17 @@ An example event for `alert_v7` looks as following: | carbon_black_cloud.alert.workflow.changed_by_type | The type of user who changed the workflow. | keyword | | carbon_black_cloud.alert.workflow.closure_reason | Reason for which the workflow was closed. | keyword | | carbon_black_cloud.alert.workflow.status | The status of the workflow. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.reason | Reason why this event happened, according to the source. This describes the why of a particular action or outcome captured in the event. Where `event.action` captures the action from the event, `event.reason` describes why that action was taken. For example, a web proxy with an `event.action` which denied the request may also populate `event.reason` with the reason why (e.g. `blocked site`). | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.mac | Host mac addresses. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.executable | Absolute path to the process executable. | keyword | -| process.executable.text | Multi-field of `process.executable`. | match_only_text | -| process.hash.md5 | MD5 hash. | keyword | -| process.hash.sha256 | SHA256 hash. | keyword | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.parent.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.parent.command_line.text | Multi-field of `process.parent.command_line`. | match_only_text | -| process.parent.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.parent.executable | Absolute path to the process executable. | keyword | -| process.parent.executable.text | Multi-field of `process.parent.executable`. | match_only_text | -| process.parent.hash.md5 | MD5 hash. | keyword | -| process.parent.hash.sha256 | SHA256 hash. | keyword | -| process.parent.name | Process name. Sometimes called program name or similar. | keyword | -| process.parent.name.text | Multi-field of `process.parent.name`. | match_only_text | -| process.parent.pid | Process id. | long | -| process.pid | Process id. | long | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | ### Endpoint Event @@ -1022,83 +888,17 @@ An example event for `endpoint_event` looks as following: | carbon_black_cloud.endpoint_event.sensor_action | The sensor action taken on event. | keyword | | carbon_black_cloud.endpoint_event.target_cmdline | Process command line associated with the target process. | keyword | | carbon_black_cloud.endpoint_event.type | The event type. | keyword | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| client.port | Port of the client. | long | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| dll.hash.md5 | MD5 hash. | keyword | -| dll.hash.sha256 | SHA256 hash. | keyword | -| dll.path | Full file path of the library. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.reason | Reason why this event happened, according to the source. This describes the why of a particular action or outcome captured in the event. Where `event.action` captures the action from the event, `event.reason` describes why that action was taken. For example, a web proxy with an `event.action` which denied the request may also populate `event.reason` with the reason why (e.g. `blocked site`). | keyword | -| file.hash.md5 | MD5 hash. | keyword | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.mac | Host mac addresses. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.executable | Absolute path to the process executable. | keyword | -| process.executable.text | Multi-field of `process.executable`. | match_only_text | -| process.hash.md5 | MD5 hash. | keyword | -| process.hash.sha256 | SHA256 hash. | keyword | -| process.parent.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.parent.command_line.text | Multi-field of `process.parent.command_line`. | match_only_text | -| process.parent.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.parent.executable | Absolute path to the process executable. | keyword | -| process.parent.executable.text | Multi-field of `process.parent.executable`. | match_only_text | -| process.parent.hash.md5 | MD5 hash. | keyword | -| process.parent.hash.sha256 | SHA256 hash. | keyword | -| process.parent.pid | Process id. | long | -| process.pid | Process id. | long | -| registry.path | Full path, including hive, key and value | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | ### Watchlist Hit @@ -1268,65 +1068,17 @@ An example event for `watchlist_hit` looks as following: | carbon_black_cloud.watchlist_hit.type | The watchlist hit type. | keyword | | carbon_black_cloud.watchlist_hit.watchlists.id | The ID of the watchlists. | keyword | | carbon_black_cloud.watchlist_hit.watchlists.name | The name of the watchlists. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.mac | Host mac addresses. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.executable | Absolute path to the process executable. | keyword | -| process.executable.text | Multi-field of `process.executable`. | match_only_text | -| process.hash.md5 | MD5 hash. | keyword | -| process.hash.sha256 | SHA256 hash. | keyword | -| process.parent.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.parent.command_line.text | Multi-field of `process.parent.command_line`. | match_only_text | -| process.parent.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.parent.executable | Absolute path to the process executable. | keyword | -| process.parent.executable.text | Multi-field of `process.parent.executable`. | match_only_text | -| process.parent.hash.md5 | MD5 hash. | keyword | -| process.parent.hash.sha256 | SHA256 hash. | keyword | -| process.parent.pid | Process id. | long | -| process.pid | Process id. | long | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | ### Asset Vulnerability Summary @@ -1427,42 +1179,14 @@ An example event for `asset_vulnerability_summary` looks as following: | carbon_black_cloud.asset_vulnerability_summary.vm.id | The identifier is for the Virtual Machine ID. | keyword | | carbon_black_cloud.asset_vulnerability_summary.vm.name | The identifier is for the Virtual Machine name. | keyword | | carbon_black_cloud.asset_vulnerability_summary.vuln_count | The identifier is for the Number of vulnerabilities at this level. | integer | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.mac | Host mac addresses. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| vulnerability.score.base | Scores can range from 0.0 to 10.0, with 10.0 being the most severe. Base scores cover an assessment for exploitability metrics (attack vector, complexity, privileges, and user interaction), impact metrics (confidentiality, integrity, and availability), and scope. For example (https://www.first.org/cvss/specification-document) | float | -| vulnerability.severity | The severity of the vulnerability can help with metrics and internal prioritization regarding remediation. For example (https://nvd.nist.gov/vuln-metrics/cvss) | keyword | diff --git a/packages/carbon_black_cloud/manifest.yml b/packages/carbon_black_cloud/manifest.yml index 6a039df6fe1..315c0c9647e 100644 --- a/packages/carbon_black_cloud/manifest.yml +++ b/packages/carbon_black_cloud/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: carbon_black_cloud title: VMware Carbon Black Cloud -version: "2.1.0" +version: "2.2.0" description: Collect logs from VMWare Carbon Black Cloud with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - edr_xdr conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" screenshots: - src: /img/carbon_black_cloud-screenshot.png title: Carbon Black Cloud alert dashboard screenshot From 70de6ca3900ba3a6c475d09c2d5c0073609e3962 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:27 +0930 Subject: [PATCH 020/121] [carbonblack_edr] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^7.14.0 || ^8.0.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/carbonblack_edr --- packages/carbonblack_edr/changelog.yml | 5 + .../data_stream/log/fields/agent.yml | 75 +----------- .../data_stream/log/fields/beats.yml | 3 - .../data_stream/log/fields/ecs.yml | 112 ------------------ packages/carbonblack_edr/docs/README.md | 74 ------------ packages/carbonblack_edr/manifest.yml | 4 +- 6 files changed, 8 insertions(+), 265 deletions(-) delete mode 100644 packages/carbonblack_edr/data_stream/log/fields/ecs.yml diff --git a/packages/carbonblack_edr/changelog.yml b/packages/carbonblack_edr/changelog.yml index 776174ec3a9..5fb93221fd3 100644 --- a/packages/carbonblack_edr/changelog.yml +++ b/packages/carbonblack_edr/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.18.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.17.0" changes: - description: Update manifest format version to v3.0.3. diff --git a/packages/carbonblack_edr/data_stream/log/fields/agent.yml b/packages/carbonblack_edr/data_stream/log/fields/agent.yml index 8d787b7c8dc..bc42d0a853b 100644 --- a/packages/carbonblack_edr/data_stream/log/fields/agent.yml +++ b/packages/carbonblack_edr/data_stream/log/fields/agent.yml @@ -1,82 +1,9 @@ - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/carbonblack_edr/data_stream/log/fields/beats.yml b/packages/carbonblack_edr/data_stream/log/fields/beats.yml index 9275638f93a..582ff946c0d 100644 --- a/packages/carbonblack_edr/data_stream/log/fields/beats.yml +++ b/packages/carbonblack_edr/data_stream/log/fields/beats.yml @@ -7,9 +7,6 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. - name: log.source.address type: keyword description: Source address from which the log event was read / sent from. diff --git a/packages/carbonblack_edr/data_stream/log/fields/ecs.yml b/packages/carbonblack_edr/data_stream/log/fields/ecs.yml deleted file mode 100644 index e490e93cb9f..00000000000 --- a/packages/carbonblack_edr/data_stream/log/fields/ecs.yml +++ /dev/null @@ -1,112 +0,0 @@ -- name: ecs.version - external: ecs -- name: error.message - external: ecs -- name: event.action - external: ecs -- name: event.category - external: ecs -- name: event.created - external: ecs -- name: event.duration - external: ecs -- name: event.end - external: ecs -- name: event.id - external: ecs -- name: event.ingested - external: ecs -- name: event.kind - external: ecs -- name: event.original - external: ecs -- name: event.outcome - external: ecs -- name: event.start - external: ecs -- name: event.type - external: ecs -- name: file.attributes - external: ecs -- name: file.code_signature.exists - external: ecs -- name: file.code_signature.status - external: ecs -- name: file.code_signature.subject_name - external: ecs -- name: file.hash.md5 - external: ecs -- name: file.path - external: ecs -- name: file.pe.architecture - external: ecs -- name: file.size - external: ecs -- name: host.name - external: ecs -- name: host.os.name - external: ecs -- name: host.os.type - external: ecs -- name: network.direction - external: ecs -- name: network.transport - external: ecs -- name: network.iana_number - external: ecs -- name: observer.name - external: ecs -- name: observer.product - external: ecs -- name: observer.type - external: ecs -- name: observer.vendor - external: ecs -- name: observer.version - external: ecs -- name: process.command_line - external: ecs -- name: process.entity_id - external: ecs -- name: process.executable - external: ecs -- name: process.hash.md5 - external: ecs -- name: process.name - external: ecs -- name: process.parent.entity_id - external: ecs -- name: process.parent.hash.md5 - external: ecs -- name: process.parent.name - external: ecs -- name: process.parent.pid - external: ecs -- name: process.pid - external: ecs -- name: process.start - external: ecs -- name: registry.path - external: ecs -- name: related.hash - external: ecs -- name: rule.id - external: ecs -- name: rule.name - external: ecs -- name: tags - external: ecs -- name: threat.indicator.type - external: ecs -- name: threat.indicator.url.domain - external: ecs -- name: threat.indicator.ip - external: ecs -- name: threat.indicator.file.hash.md5 - external: ecs -- name: threat.indicator.port - external: ecs -- name: tls.client.ja3 - external: ecs -- name: tls.server.ja3s - external: ecs diff --git a/packages/carbonblack_edr/docs/README.md b/packages/carbonblack_edr/docs/README.md index 91830774746..da604b58305 100644 --- a/packages/carbonblack_edr/docs/README.md +++ b/packages/carbonblack_edr/docs/README.md @@ -264,88 +264,14 @@ An example event for `log` looks as following: | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.duration | Duration of the event in nanoseconds. If `event.start` and `event.end` are known this value should be the difference between the end and start time. | long | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.attributes | Array of file attributes. Attributes names will vary by platform. Here's a non-exhaustive list of values that are expected in this field: archive, compressed, directory, encrypted, execute, hidden, read, readonly, system, write. | keyword | -| file.code_signature.exists | Boolean to capture if a signature is present. | boolean | -| file.code_signature.status | Additional information about the certificate status. This is useful for logging cryptographic errors with the certificate validity or trust status. Leave unpopulated if the validity or trust of the certificate was unchecked. | keyword | -| file.code_signature.subject_name | Subject name of the code signer | keyword | -| file.hash.md5 | MD5 hash. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| file.pe.architecture | CPU architecture target for the file. | keyword | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.type | Use the `os.type` field to categorize the operating system into one of the broad commercial families. If the OS you're dealing with is not listed as an expected value, the field should not be populated. Please let us know by opening an issue with ECS, to propose its addition. | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| observer.name | Custom name of the observer. This is a name that can be given to an observer. This can be helpful for example if multiple firewalls of the same model are used in an organization. If no custom name is needed, the field can be left empty. | keyword | -| observer.product | The product name of the observer. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| observer.version | Observer version. | keyword | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.executable | Absolute path to the process executable. | keyword | -| process.executable.text | Multi-field of `process.executable`. | match_only_text | -| process.hash.md5 | MD5 hash. | keyword | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.parent.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.parent.hash.md5 | MD5 hash. | keyword | -| process.parent.name | Process name. Sometimes called program name or similar. | keyword | -| process.parent.name.text | Multi-field of `process.parent.name`. | match_only_text | -| process.parent.pid | Process id. | long | -| process.pid | Process id. | long | -| process.start | The time the process started. | date | -| registry.path | Full path, including hive, key and value | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | -| threat.indicator.port | Identifies a threat indicator as a port number (irrespective of direction). | long | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| tls.client.ja3 | A hash that identifies clients based on how they perform an SSL/TLS handshake. | keyword | -| tls.server.ja3s | A hash that identifies servers based on how they perform an SSL/TLS handshake. | keyword | diff --git a/packages/carbonblack_edr/manifest.yml b/packages/carbonblack_edr/manifest.yml index 97243c9291e..840d63ed13d 100644 --- a/packages/carbonblack_edr/manifest.yml +++ b/packages/carbonblack_edr/manifest.yml @@ -1,13 +1,13 @@ name: carbonblack_edr title: VMware Carbon Black EDR -version: "1.17.0" +version: "1.18.0" description: Collect logs from VMware Carbon Black EDR with Elastic Agent. type: integration format_version: "3.0.3" categories: [security, edr_xdr] conditions: kibana: - version: ^7.14.0 || ^8.0.0 + version: "^8.13.0" policy_templates: - name: log title: Carbon Black EDR logs From 318ee01a15ac663efadbc610363754f80e1adef5 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:28 +0930 Subject: [PATCH 021/121] [cel] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. The ecs.version in sample_event.json files was changed to 8.11.0. Previously sample_event.json files contained 8.0.0. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/cel --- packages/cel/changelog.yml | 5 +++++ packages/cel/fields/input.yml | 9 --------- packages/cel/manifest.yml | 8 +++----- packages/cel/sample_event.json | 2 +- 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/packages/cel/changelog.yml b/packages/cel/changelog.yml index 6e834e8c630..8126f0be8ea 100644 --- a/packages/cel/changelog.yml +++ b/packages/cel/changelog.yml @@ -1,3 +1,8 @@ +- version: "1.12.0" + changes: + - description: ECS version updated to 8.11.0. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.11.0" changes: - description: Enable use of Digest Authentication. diff --git a/packages/cel/fields/input.yml b/packages/cel/fields/input.yml index 0830dd1e248..3f2775fe36b 100644 --- a/packages/cel/fields/input.yml +++ b/packages/cel/fields/input.yml @@ -1,9 +1,5 @@ - name: "@timestamp" external: ecs -- name: ecs.version - external: ecs -- name: message - external: ecs - name: input.name type: constant_keyword - name: input.type @@ -21,8 +17,3 @@ external: ecs type: constant_keyword value: cel -- name: event.dataset - external: ecs - type: constant_keyword -- name: tags - external: ecs diff --git a/packages/cel/manifest.yml b/packages/cel/manifest.yml index 21734a919f7..db203af2bfe 100644 --- a/packages/cel/manifest.yml +++ b/packages/cel/manifest.yml @@ -3,12 +3,12 @@ name: cel title: Custom API using Common Expression Language description: Collect custom events from an API with Elastic agent type: input -version: "1.11.0" +version: "1.12.0" categories: - custom conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" elastic: subscription: "basic" policy_templates: @@ -118,7 +118,7 @@ policy_templates: secret: true - name: digest_no_reuse type: bool - title: Digest No Challenge Reuse + title: Digest No Challenge Reuse show_user: true required: false description: Selecting no challenge reuse prevents the transport from reusing digest challenges @@ -337,7 +337,6 @@ policy_templates: show_user: false description: > Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. - - name: tags type: text title: Tags @@ -353,7 +352,6 @@ policy_templates: show_user: false description: > The request tracer logs HTTP requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-cel.html#_resource_tracer_filename) for details. - owner: github: elastic/security-service-integrations type: elastic diff --git a/packages/cel/sample_event.json b/packages/cel/sample_event.json index 8e26cb7756f..396bb0d6899 100644 --- a/packages/cel/sample_event.json +++ b/packages/cel/sample_event.json @@ -13,7 +13,7 @@ "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "8c8782fa-cd5b-4ae8-94a0-ee8e3ea9a8df", From 3e0aa348850b17c9b8f4fef4c15ae8813a6f1d7c Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:28 +0930 Subject: [PATCH 022/121] [cisa_kevs] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.11.4 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/cisa_kevs --- packages/cisa_kevs/changelog.yml | 7 +++++- .../data_stream/vulnerability/fields/ecs.yml | 24 ------------------- packages/cisa_kevs/docs/README.md | 13 ---------- packages/cisa_kevs/manifest.yml | 4 ++-- 4 files changed, 8 insertions(+), 40 deletions(-) delete mode 100644 packages/cisa_kevs/data_stream/vulnerability/fields/ecs.yml diff --git a/packages/cisa_kevs/changelog.yml b/packages/cisa_kevs/changelog.yml index 8ed89383aff..30952d1b239 100644 --- a/packages/cisa_kevs/changelog.yml +++ b/packages/cisa_kevs/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.2.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.1.0" changes: - description: Improve handling of empty responses. @@ -6,7 +11,7 @@ link: https://github.com/elastic/integrations/pull/9974 - version: "1.0.1" changes: - - description: Update logo to align w/ Elastic Integrations page, fix description wording + - description: Update logo to align w/ Elastic Integrations page, fix description wording type: bugfix link: https://github.com/elastic/integrations/pull/9631 - version: "1.0.0" diff --git a/packages/cisa_kevs/data_stream/vulnerability/fields/ecs.yml b/packages/cisa_kevs/data_stream/vulnerability/fields/ecs.yml deleted file mode 100644 index c8f59ebea1b..00000000000 --- a/packages/cisa_kevs/data_stream/vulnerability/fields/ecs.yml +++ /dev/null @@ -1,24 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: error.message -- external: ecs - name: tags -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.category -- external: ecs - name: event.type -- external: ecs - name: event.created -- external: ecs - name: event.original -- external: ecs - name: vulnerability.id -- external: ecs - name: vulnerability.description diff --git a/packages/cisa_kevs/docs/README.md b/packages/cisa_kevs/docs/README.md index 662f82be7aa..ab11a409701 100644 --- a/packages/cisa_kevs/docs/README.md +++ b/packages/cisa_kevs/docs/README.md @@ -125,17 +125,4 @@ An example event for `vulnerability` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | input.type | Type of Filebeat input. | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| vulnerability.description | The description of the vulnerability that provides additional context of the vulnerability. For example (https://cve.mitre.org/about/faqs.html#cve_entry_descriptions_created[Common Vulnerabilities and Exposure CVE description]) | keyword | -| vulnerability.description.text | Multi-field of `vulnerability.description`. | match_only_text | -| vulnerability.id | The identification (ID) is the number portion of a vulnerability entry. It includes a unique identification number for the vulnerability. For example (https://cve.mitre.org/about/faqs.html#what_is_cve_id)[Common Vulnerabilities and Exposure CVE ID] | keyword | diff --git a/packages/cisa_kevs/manifest.yml b/packages/cisa_kevs/manifest.yml index 9d39055e51f..02798ef3db9 100644 --- a/packages/cisa_kevs/manifest.yml +++ b/packages/cisa_kevs/manifest.yml @@ -1,14 +1,14 @@ format_version: 3.0.3 name: cisa_kevs title: "CISA Known Exploited Vulnerabilities" -version: 1.1.0 +version: "1.2.0" description: "This package allows the ingest of known exploited vulnerabilities according to the Cybersecurity and Infrastructure Security Agency of the United States of America. This information could be used to enrich or track exisiting vulnerabilities that are known to be exploited in the wild." type: integration categories: - security conditions: kibana: - version: "^8.11.4" + version: "^8.13.0" elastic: subscription: "basic" screenshots: From d18fee236aa29ddc0333992111223572ff91aafa Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:32 +0930 Subject: [PATCH 023/121] [cisco_duo] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/cisco_duo --- packages/cisco_duo/changelog.yml | 5 + .../data_stream/admin/fields/agent.yml | 147 ------------ .../data_stream/admin/fields/ecs.yml | 40 ---- .../data_stream/auth/fields/agent.yml | 147 ------------ .../cisco_duo/data_stream/auth/fields/ecs.yml | 72 ------ .../offline_enrollment/fields/agent.yml | 147 ------------ .../offline_enrollment/fields/ecs.yml | 14 -- .../pipeline/test-summary.log-expected.json | 4 +- .../data_stream/summary/fields/agent.yml | 147 ------------ .../data_stream/summary/fields/ecs.yml | 8 - .../data_stream/telephony/fields/agent.yml | 147 ------------ .../data_stream/telephony/fields/ecs.yml | 10 - packages/cisco_duo/docs/README.md | 211 ------------------ packages/cisco_duo/manifest.yml | 4 +- 14 files changed, 9 insertions(+), 1094 deletions(-) delete mode 100644 packages/cisco_duo/data_stream/admin/fields/ecs.yml delete mode 100644 packages/cisco_duo/data_stream/auth/fields/ecs.yml delete mode 100644 packages/cisco_duo/data_stream/offline_enrollment/fields/ecs.yml delete mode 100644 packages/cisco_duo/data_stream/summary/fields/ecs.yml delete mode 100644 packages/cisco_duo/data_stream/telephony/fields/ecs.yml diff --git a/packages/cisco_duo/changelog.yml b/packages/cisco_duo/changelog.yml index ff303fb2492..f78aa3184a8 100644 --- a/packages/cisco_duo/changelog.yml +++ b/packages/cisco_duo/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.24.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.23.0" changes: - description: Improve error handling. diff --git a/packages/cisco_duo/data_stream/admin/fields/agent.yml b/packages/cisco_duo/data_stream/admin/fields/agent.yml index 215021047d4..f833857d0fe 100644 --- a/packages/cisco_duo/data_stream/admin/fields/agent.yml +++ b/packages/cisco_duo/data_stream/admin/fields/agent.yml @@ -5,162 +5,15 @@ footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." - - name: id - level: core - type: keyword - ignore_above: 1024 - description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." - name: containerized type: boolean description: >- diff --git a/packages/cisco_duo/data_stream/admin/fields/ecs.yml b/packages/cisco_duo/data_stream/admin/fields/ecs.yml deleted file mode 100644 index ee7c0848dbb..00000000000 --- a/packages/cisco_duo/data_stream/admin/fields/ecs.yml +++ /dev/null @@ -1,40 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.agent_id_status -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.reason -- external: ecs - name: event.type -- external: ecs - name: message -- external: ecs - name: related.user -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: tags -- external: ecs - name: user.changes.email -- external: ecs - name: user.changes.name -- external: ecs - name: user.email -- external: ecs - name: user.name -- external: ecs - name: user.target.name diff --git a/packages/cisco_duo/data_stream/auth/fields/agent.yml b/packages/cisco_duo/data_stream/auth/fields/agent.yml index 215021047d4..f833857d0fe 100644 --- a/packages/cisco_duo/data_stream/auth/fields/agent.yml +++ b/packages/cisco_duo/data_stream/auth/fields/agent.yml @@ -5,162 +5,15 @@ footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." - - name: id - level: core - type: keyword - ignore_above: 1024 - description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." - name: containerized type: boolean description: >- diff --git a/packages/cisco_duo/data_stream/auth/fields/ecs.yml b/packages/cisco_duo/data_stream/auth/fields/ecs.yml deleted file mode 100644 index 1915d7b7ccc..00000000000 --- a/packages/cisco_duo/data_stream/auth/fields/ecs.yml +++ /dev/null @@ -1,72 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.agent_id_status -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.reason -- external: ecs - name: event.type -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: source.address -- external: ecs - name: source.user.email -- external: ecs - name: source.user.id -- external: ecs - name: source.user.name -- external: ecs - name: source.user.group.name -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: tags -- external: ecs - name: user.email -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version diff --git a/packages/cisco_duo/data_stream/offline_enrollment/fields/agent.yml b/packages/cisco_duo/data_stream/offline_enrollment/fields/agent.yml index 215021047d4..f833857d0fe 100644 --- a/packages/cisco_duo/data_stream/offline_enrollment/fields/agent.yml +++ b/packages/cisco_duo/data_stream/offline_enrollment/fields/agent.yml @@ -5,162 +5,15 @@ footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." - - name: id - level: core - type: keyword - ignore_above: 1024 - description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." - name: containerized type: boolean description: >- diff --git a/packages/cisco_duo/data_stream/offline_enrollment/fields/ecs.yml b/packages/cisco_duo/data_stream/offline_enrollment/fields/ecs.yml deleted file mode 100644 index 8082727b549..00000000000 --- a/packages/cisco_duo/data_stream/offline_enrollment/fields/ecs.yml +++ /dev/null @@ -1,14 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.created -- external: ecs - name: event.original -- external: ecs - name: related.hosts -- external: ecs - name: related.user -- external: ecs - name: tags -- external: ecs - name: user.name diff --git a/packages/cisco_duo/data_stream/summary/_dev/test/pipeline/test-summary.log-expected.json b/packages/cisco_duo/data_stream/summary/_dev/test/pipeline/test-summary.log-expected.json index 20dd9e8fae5..5e8c15d22ea 100644 --- a/packages/cisco_duo/data_stream/summary/_dev/test/pipeline/test-summary.log-expected.json +++ b/packages/cisco_duo/data_stream/summary/_dev/test/pipeline/test-summary.log-expected.json @@ -1,7 +1,7 @@ { "expected": [ { - "@timestamp": "2024-05-12T22:53:11.467044414Z", + "@timestamp": "2024-06-20T03:53:31.527976915Z", "cisco_duo": { "summary": { "admin_count": 6, @@ -21,7 +21,7 @@ ] }, { - "@timestamp": "2024-05-12T22:53:11.467103248Z", + "@timestamp": "2024-06-20T03:53:31.527986750Z", "cisco_duo": { "summary": { "admin_count": 3, diff --git a/packages/cisco_duo/data_stream/summary/fields/agent.yml b/packages/cisco_duo/data_stream/summary/fields/agent.yml index 215021047d4..f833857d0fe 100644 --- a/packages/cisco_duo/data_stream/summary/fields/agent.yml +++ b/packages/cisco_duo/data_stream/summary/fields/agent.yml @@ -5,162 +5,15 @@ footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." - - name: id - level: core - type: keyword - ignore_above: 1024 - description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." - name: containerized type: boolean description: >- diff --git a/packages/cisco_duo/data_stream/summary/fields/ecs.yml b/packages/cisco_duo/data_stream/summary/fields/ecs.yml deleted file mode 100644 index b334910e040..00000000000 --- a/packages/cisco_duo/data_stream/summary/fields/ecs.yml +++ /dev/null @@ -1,8 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.created -- external: ecs - name: event.original -- external: ecs - name: tags diff --git a/packages/cisco_duo/data_stream/telephony/fields/agent.yml b/packages/cisco_duo/data_stream/telephony/fields/agent.yml index 215021047d4..f833857d0fe 100644 --- a/packages/cisco_duo/data_stream/telephony/fields/agent.yml +++ b/packages/cisco_duo/data_stream/telephony/fields/agent.yml @@ -5,162 +5,15 @@ footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." - - name: id - level: core - type: keyword - ignore_above: 1024 - description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." - name: containerized type: boolean description: >- diff --git a/packages/cisco_duo/data_stream/telephony/fields/ecs.yml b/packages/cisco_duo/data_stream/telephony/fields/ecs.yml deleted file mode 100644 index a24c00497b5..00000000000 --- a/packages/cisco_duo/data_stream/telephony/fields/ecs.yml +++ /dev/null @@ -1,10 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: tags diff --git a/packages/cisco_duo/docs/README.md b/packages/cisco_duo/docs/README.md index fac374627c3..de8bc065feb 100644 --- a/packages/cisco_duo/docs/README.md +++ b/packages/cisco_duo/docs/README.md @@ -102,67 +102,17 @@ An example event for `admin` looks as following: | cisco_duo.admin.action_performed_on | The object that was acted on. | keyword | | cisco_duo.admin.flattened | ES flattened datatype for objects where the subfields aren't known in advance. | flattened | | cisco_duo.admin.user.name | The full name of the administrator who performed the action in the Duo Admin Panel. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.agent_id_status | Agents are normally responsible for populating the `agent.id` field value. If the system receiving events is capable of validating the value based on authentication information for the client then this field can be used to reflect the outcome of that validation. For example if the agent's connection is authenticated with mTLS and the client cert contains the ID of the agent to which the cert was issued then the `agent.id` value in events can be checked against the certificate. If the values match then `event.agent_id_status: verified` is added to the event, otherwise one of the other allowed values should be used. If no validation is performed then the field should be omitted. The allowed values are: `verified` - The `agent.id` field value matches expected value obtained from auth metadata. `mismatch` - The `agent.id` field value does not match the expected value obtained from auth metadata. `missing` - There was no `agent.id` field in the event to validate. `auth_metadata_missing` - There was no auth metadata or it was missing information about the agent ID. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.reason | Reason why this event happened, according to the source. This describes the why of a particular action or outcome captured in the event. Where `event.action` captures the action from the event, `event.reason` describes why that action was taken. For example, a web proxy with an `event.action` which denied the request may also populate `event.reason` with the reason why (e.g. `blocked site`). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| user.changes.email | User email address. | keyword | -| user.changes.name | Short name or login of the user. | keyword | -| user.changes.name.text | Multi-field of `user.changes.name`. | match_only_text | -| user.email | User email address. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user.target.name | Short name or login of the user. | keyword | -| user.target.name.text | Multi-field of `user.target.name`. | match_only_text | ### Authentication @@ -360,83 +310,17 @@ An example event for `auth` looks as following: | cisco_duo.auth.result | The result of the authentication attempt. | keyword | | cisco_duo.auth.trusted_endpoint_status | Status of Trusted Endpoint. | keyword | | cisco_duo.auth.txid | The transaction ID of the event. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.agent_id_status | Agents are normally responsible for populating the `agent.id` field value. If the system receiving events is capable of validating the value based on authentication information for the client then this field can be used to reflect the outcome of that validation. For example if the agent's connection is authenticated with mTLS and the client cert contains the ID of the agent to which the cert was issued then the `agent.id` value in events can be checked against the certificate. If the values match then `event.agent_id_status: verified` is added to the event, otherwise one of the other allowed values should be used. If no validation is performed then the field should be omitted. The allowed values are: `verified` - The `agent.id` field value matches expected value obtained from auth metadata. `mismatch` - The `agent.id` field value does not match the expected value obtained from auth metadata. `missing` - There was no `agent.id` field in the event to validate. `auth_metadata_missing` - There was no auth metadata or it was missing information about the agent ID. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.reason | Reason why this event happened, according to the source. This describes the why of a particular action or outcome captured in the event. Where `event.action` captures the action from the event, `event.reason` describes why that action was taken. For example, a web proxy with an `event.action` which denied the request may also populate `event.reason` with the reason why (e.g. `blocked site`). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| source.user.email | User email address. | keyword | -| source.user.group.name | Name of the group. | keyword | -| source.user.id | Unique identifier of the user. | keyword | -| source.user.name | Short name or login of the user. | keyword | -| source.user.name.text | Multi-field of `source.user.name`. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | ### Offline Enrollment @@ -523,51 +407,17 @@ An example event for `offline_enrollment` looks as following: | cisco_duo.offline_enrollment.description.user_agent | The Duo Windows Logon application version information and the Windows OS version and platform information. | keyword | | cisco_duo.offline_enrollment.object | The Duo Windows Logon integration's name. | keyword | | cisco_duo.offline_enrollment.user.name | The Duo username | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | ### Summary @@ -635,47 +485,17 @@ An example event for `summary` looks as following: | cisco_duo.summary.integration_count | Current number of integrations in the account. | integer | | cisco_duo.summary.telephony_credits_remaining | Current total number of telephony credits available in the account. This is the sum of all types of telephony credits. | integer | | cisco_duo.summary.user_count | Current number of users in the account. | integer | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| tags | List of keywords used to tag each event. | keyword | ### Telephony @@ -744,45 +564,14 @@ An example event for `telephony` looks as following: | cisco_duo.telephony.event_type | How this telephony event was initiated. | keyword | | cisco_duo.telephony.phone_number | The phone number that initiated this event. | keyword | | cisco_duo.telephony.type | This type of telephony Event. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| tags | List of keywords used to tag each event. | keyword | diff --git a/packages/cisco_duo/manifest.yml b/packages/cisco_duo/manifest.yml index d2f98281134..8d5f297db56 100644 --- a/packages/cisco_duo/manifest.yml +++ b/packages/cisco_duo/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: cisco_duo title: Cisco Duo -version: "1.23.0" +version: "1.24.0" description: Collect logs from Cisco Duo with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - iam conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" screenshots: - src: /img/cisco_duo-screenshot.png title: Cisco Duo authentication log dashboard From bea8c1d290c2f7354efa0a63b39b06cb0d932e68 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:36 +0930 Subject: [PATCH 024/121] [cisco_meraki] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/cisco_meraki --- packages/cisco_meraki/changelog.yml | 5 + .../test/system/test-meraki-https-config.yml | 2 +- .../data_stream/events/fields/agent.yml | 139 +------ .../data_stream/events/fields/base-fields.yml | 4 - .../data_stream/events/fields/ecs.yml | 246 ------------- .../log/_dev/test/system/test-udp-config.yml | 10 +- .../data_stream/log/fields/agent.yml | 139 +------ .../data_stream/log/fields/base-fields.yml | 4 - .../data_stream/log/fields/ecs.yml | 296 --------------- packages/cisco_meraki/docs/README.md | 341 ------------------ packages/cisco_meraki/manifest.yml | 4 +- 11 files changed, 15 insertions(+), 1175 deletions(-) diff --git a/packages/cisco_meraki/changelog.yml b/packages/cisco_meraki/changelog.yml index 49a815c257b..b25e86da52e 100644 --- a/packages/cisco_meraki/changelog.yml +++ b/packages/cisco_meraki/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.23.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.22.0" changes: - description: Retain message for all events. diff --git a/packages/cisco_meraki/data_stream/events/_dev/test/system/test-meraki-https-config.yml b/packages/cisco_meraki/data_stream/events/_dev/test/system/test-meraki-https-config.yml index 062f0400af8..af321ff7ea2 100644 --- a/packages/cisco_meraki/data_stream/events/_dev/test/system/test-meraki-https-config.yml +++ b/packages/cisco_meraki/data_stream/events/_dev/test/system/test-meraki-https-config.yml @@ -61,4 +61,4 @@ data_stream: -----END PRIVATE KEY----- verification_mode: none assert: - hit_count: 2 \ No newline at end of file + hit_count: 2 diff --git a/packages/cisco_meraki/data_stream/events/fields/agent.yml b/packages/cisco_meraki/data_stream/events/fields/agent.yml index 4c4f4b2d93a..b1694c35c8a 100644 --- a/packages/cisco_meraki/data_stream/events/fields/agent.yml +++ b/packages/cisco_meraki/data_stream/events/fields/agent.yml @@ -5,152 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cisco_meraki/data_stream/events/fields/base-fields.yml b/packages/cisco_meraki/data_stream/events/fields/base-fields.yml index fcbdca9da69..71da0e30206 100644 --- a/packages/cisco_meraki/data_stream/events/fields/base-fields.yml +++ b/packages/cisco_meraki/data_stream/events/fields/base-fields.yml @@ -15,7 +15,3 @@ type: constant_keyword description: Event dataset value: cisco_meraki.events -- name: container.id - description: Unique container id. - ignore_above: 1024 - type: keyword diff --git a/packages/cisco_meraki/data_stream/events/fields/ecs.yml b/packages/cisco_meraki/data_stream/events/fields/ecs.yml index 4a063f496f5..adb0dc85322 100644 --- a/packages/cisco_meraki/data_stream/events/fields/ecs.yml +++ b/packages/cisco_meraki/data_stream/events/fields/ecs.yml @@ -1,248 +1,2 @@ - external: ecs name: '@timestamp' -- external: ecs - name: client.ip -- external: ecs - name: client.mac -- external: ecs - name: client.domain -- external: ecs - name: client.registered_domain -- external: ecs - name: client.subdomain -- external: ecs - name: client.top_level_domain -- external: ecs - name: destination.address -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.bytes -- external: ecs - name: destination.domain -- external: ecs - name: destination.ip -- external: ecs - name: destination.mac -- external: ecs - name: destination.nat.ip -- external: ecs - name: destination.nat.port -- external: ecs - name: destination.port -- external: ecs - name: destination.registered_domain -- external: ecs - name: destination.subdomain -- external: ecs - name: destination.top_level_domain -- external: ecs - name: dns.answers.name -- external: ecs - name: dns.answers.type -- external: ecs - name: dns.question.registered_domain -- external: ecs - name: dns.question.subdomain -- external: ecs - name: dns.question.top_level_domain -- external: ecs - name: dns.question.type -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.code -- external: ecs - name: event.ingested -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.timezone -- external: ecs - name: event.type -- external: ecs - name: event.category -- external: ecs - name: file.attributes -- external: ecs - name: file.directory -- external: ecs - name: file.extension -- external: ecs - name: file.name -- external: ecs - name: file.path -- external: ecs - name: file.size -- external: ecs - name: file.type -- external: ecs - name: group.id -- external: ecs - name: group.name -- external: ecs - name: host.hostname -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: http.request.method -- external: ecs - name: http.request.referrer -- external: ecs - name: log.level -- external: ecs - name: log.syslog.facility.code -- external: ecs - name: log.syslog.priority -- external: ecs - name: log.syslog.severity.code -- external: ecs - name: log.file.path -- external: ecs - name: tags -- external: ecs - name: message -- external: ecs - name: network.application -- external: ecs - name: network.bytes -- external: ecs - name: network.direction -- external: ecs - name: network.forwarded_ip -- external: ecs - name: network.packets -- external: ecs - name: network.protocol -- external: ecs - name: observer.egress.interface.name -- external: ecs - name: observer.ingress.interface.name -- external: ecs - name: observer.product -- external: ecs - name: observer.type -- external: ecs - name: observer.vendor -- external: ecs - name: observer.version -- external: ecs - name: observer.mac -- external: ecs - name: observer.name -- external: ecs - name: observer.serial_number -- external: ecs - name: process.name -- external: ecs - name: process.parent.name -- external: ecs - name: process.parent.title -- external: ecs - name: process.pid -- external: ecs - name: process.parent.pid -- external: ecs - name: process.title -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: rule.name -- external: ecs - name: server.mac -- external: ecs - name: server.domain -- external: ecs - name: server.registered_domain -- external: ecs - name: server.subdomain -- external: ecs - name: server.top_level_domain -- external: ecs - name: service.name -- external: ecs - name: source.address -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.bytes -- external: ecs - name: source.domain -- external: ecs - name: source.ip -- external: ecs - name: source.mac -- external: ecs - name: source.nat.ip -- external: ecs - name: source.nat.port -- external: ecs - name: source.port -- external: ecs - name: source.registered_domain -- external: ecs - name: source.subdomain -- external: ecs - name: source.top_level_domain -- external: ecs - name: url.domain -- external: ecs - name: url.original -- external: ecs - name: url.path -- external: ecs - name: url.query -- external: ecs - name: url.registered_domain -- external: ecs - name: url.top_level_domain -- external: ecs - name: user.domain -- external: ecs - name: user.full_name -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user_agent.original -- external: ecs - name: observer.hostname -- external: ecs - name: network.vlan.id -- external: ecs - name: threat.software.type -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.indicator.description -- external: ecs - name: threat.indicator.reference -- external: ecs - name: threat.indicator.file.name -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: organization.id -- external: ecs - name: organization.name -- external: ecs - name: network.name diff --git a/packages/cisco_meraki/data_stream/log/_dev/test/system/test-udp-config.yml b/packages/cisco_meraki/data_stream/log/_dev/test/system/test-udp-config.yml index e7918a6aa12..10b4cafe498 100644 --- a/packages/cisco_meraki/data_stream/log/_dev/test/system/test-udp-config.yml +++ b/packages/cisco_meraki/data_stream/log/_dev/test/system/test-udp-config.yml @@ -6,8 +6,8 @@ data_stream: listen_address: 0.0.0.0 listen_port: 8685 preserve_original_event: true -# Do not assert hit count for this input. Locally, the constraint is -# satisfied, but on CI, apparently the UDP input drops too many (>0) -# messages. -# assert: -# hit_count: 204 \ No newline at end of file + # Do not assert hit count for this input. Locally, the constraint is + # satisfied, but on CI, apparently the UDP input drops too many (>0) + # messages. + # assert: + # hit_count: 204 diff --git a/packages/cisco_meraki/data_stream/log/fields/agent.yml b/packages/cisco_meraki/data_stream/log/fields/agent.yml index 4c4f4b2d93a..b1694c35c8a 100644 --- a/packages/cisco_meraki/data_stream/log/fields/agent.yml +++ b/packages/cisco_meraki/data_stream/log/fields/agent.yml @@ -5,152 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cisco_meraki/data_stream/log/fields/base-fields.yml b/packages/cisco_meraki/data_stream/log/fields/base-fields.yml index 79eddd5d6c2..57cd7d544ae 100644 --- a/packages/cisco_meraki/data_stream/log/fields/base-fields.yml +++ b/packages/cisco_meraki/data_stream/log/fields/base-fields.yml @@ -15,7 +15,3 @@ type: constant_keyword description: Event dataset value: cisco_meraki.log -- name: container.id - description: Unique container id. - ignore_above: 1024 - type: keyword diff --git a/packages/cisco_meraki/data_stream/log/fields/ecs.yml b/packages/cisco_meraki/data_stream/log/fields/ecs.yml index c44c63e7226..adb0dc85322 100644 --- a/packages/cisco_meraki/data_stream/log/fields/ecs.yml +++ b/packages/cisco_meraki/data_stream/log/fields/ecs.yml @@ -1,298 +1,2 @@ - external: ecs name: '@timestamp' -- external: ecs - name: client.as.number -- external: ecs - name: client.as.organization.name -- external: ecs - name: client.ip -- external: ecs - name: client.mac -- external: ecs - name: client.domain -- external: ecs - name: client.registered_domain -- external: ecs - name: client.subdomain -- external: ecs - name: client.top_level_domain -- external: ecs - name: destination.address -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.bytes -- external: ecs - name: destination.domain -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.ip -- external: ecs - name: destination.mac -- external: ecs - name: destination.nat.ip -- external: ecs - name: destination.nat.port -- external: ecs - name: destination.port -- external: ecs - name: destination.registered_domain -- external: ecs - name: destination.subdomain -- external: ecs - name: destination.top_level_domain -- external: ecs - name: dns.answers.name -- external: ecs - name: dns.answers.type -- external: ecs - name: dns.question.registered_domain -- external: ecs - name: dns.question.subdomain -- external: ecs - name: dns.question.top_level_domain -- external: ecs - name: dns.question.type -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.code -- external: ecs - name: event.ingested -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.timezone -- external: ecs - name: event.category -- external: ecs - name: event.type -- external: ecs - name: file.attributes -- external: ecs - name: file.directory -- external: ecs - name: file.extension -- external: ecs - name: file.hash.sha256 -- external: ecs - name: file.name -- external: ecs - name: file.path -- external: ecs - name: file.size -- external: ecs - name: file.type -- external: ecs - name: group.id -- external: ecs - name: group.name -- external: ecs - name: host.hostname -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: http.request.method -- external: ecs - name: http.request.referrer -- external: ecs - name: log.level -- external: ecs - name: log.syslog.facility.code -- external: ecs - name: log.syslog.priority -- external: ecs - name: log.syslog.severity.code -- external: ecs - name: log.file.path -- external: ecs - name: tags -- external: ecs - name: message -- external: ecs - name: network.application -- external: ecs - name: network.bytes -- external: ecs - name: network.direction -- external: ecs - name: network.forwarded_ip -- external: ecs - name: network.name -- external: ecs - name: network.packets -- external: ecs - name: network.protocol -- external: ecs - name: observer.egress.interface.name -- external: ecs - name: observer.ingress.interface.name -- external: ecs - name: observer.ingress.vlan.id -- external: ecs - name: observer.product -- external: ecs - name: observer.type -- external: ecs - name: observer.vendor -- external: ecs - name: observer.version -- external: ecs - name: observer.mac -- external: ecs - name: process.name -- external: ecs - name: process.parent.name -- external: ecs - name: process.parent.title -- external: ecs - name: process.pid -- external: ecs - name: process.parent.pid -- external: ecs - name: process.title -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: rule.name -- external: ecs - name: server.mac -- external: ecs - name: server.ip -- external: ecs - name: server.domain -- external: ecs - name: server.registered_domain -- external: ecs - name: server.subdomain -- external: ecs - name: server.top_level_domain -- external: ecs - name: service.name -- external: ecs - name: source.address -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.bytes -- external: ecs - name: source.domain -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.ip -- external: ecs - name: source.mac -- external: ecs - name: source.nat.ip -- external: ecs - name: source.nat.port -- external: ecs - name: source.port -- external: ecs - name: source.registered_domain -- external: ecs - name: source.subdomain -- external: ecs - name: source.top_level_domain -- external: ecs - name: url.extension -- external: ecs - name: url.domain -- external: ecs - name: url.original -- external: ecs - name: url.path -- external: ecs - name: url.query -- external: ecs - name: url.registered_domain -- external: ecs - name: url.scheme -- external: ecs - name: url.subdomain -- external: ecs - name: url.top_level_domain -- external: ecs - name: user.domain -- external: ecs - name: user.full_name -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.version -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.os.full -- external: ecs - name: observer.hostname -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: network.vlan.id -- external: ecs - name: client.geo.city_name -- external: ecs - name: client.geo.continent_name -- external: ecs - name: client.geo.country_iso_code -- external: ecs - name: client.geo.country_name -- external: ecs - name: client.geo.location -- external: ecs - name: client.geo.region_iso_code -- external: ecs - name: client.geo.region_name diff --git a/packages/cisco_meraki/docs/README.md b/packages/cisco_meraki/docs/README.md index 7c09dbd5e47..22457755d29 100644 --- a/packages/cisco_meraki/docs/README.md +++ b/packages/cisco_meraki/docs/README.md @@ -115,202 +115,18 @@ The `cisco_meraki.log` dataset provides events from the configured syslog server | cisco_meraki.vap | | keyword | | cisco_meraki.wpa_auth | | flattened | | cisco_meraki.wpa_deauth | | flattened | -| client.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| client.as.organization.name | Organization name. | keyword | -| client.as.organization.name.text | Multi-field of `client.as.organization.name`. | match_only_text | -| client.domain | The domain name of the client system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| client.geo.city_name | City name. | keyword | -| client.geo.continent_name | Name of the continent. | keyword | -| client.geo.country_iso_code | Country ISO code. | keyword | -| client.geo.country_name | Country name. | keyword | -| client.geo.location | Longitude and latitude. | geo_point | -| client.geo.region_iso_code | Region ISO code. | keyword | -| client.geo.region_name | Region name. | keyword | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| client.mac | MAC address of the client. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| client.registered_domain | The highest registered client domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| client.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| client.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.bytes | Bytes sent from the destination to the source. | long | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.mac | MAC address of the destination. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| destination.nat.ip | Translated ip of destination based NAT sessions (e.g. internet to private DMZ) Typically used with load balancers, firewalls, or routers. | ip | -| destination.nat.port | Port the source session is translated to by NAT Device. Typically used with load balancers, firewalls, or routers. | long | -| destination.port | Port of the destination. | long | -| destination.registered_domain | The highest registered destination domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| destination.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| destination.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| dns.answers.name | The domain name to which this resource record pertains. If a chain of CNAME is being resolved, each answer's `name` should be the one that corresponds with the answer's `data`. It should not simply be the original `question.name` repeated. | keyword | -| dns.answers.type | The type of data contained in this resource record. | keyword | -| dns.question.registered_domain | The highest registered domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| dns.question.subdomain | The subdomain is all of the labels under the registered_domain. If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| dns.question.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| dns.question.type | The type of record being queried. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.attributes | Array of file attributes. Attributes names will vary by platform. Here's a non-exhaustive list of values that are expected in this field: archive, compressed, directory, encrypted, execute, hidden, read, readonly, system, write. | keyword | -| file.directory | Directory where the file is located. It should include the drive letter, when appropriate. | keyword | -| file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| file.type | File type (file, dir, or symlink). | keyword | -| group.id | Unique identifier for the group on the system/platform. | keyword | -| group.name | Name of the group. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.request.referrer | Referrer for this HTTP request. | keyword | | input.type | Input type. | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | -| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | | log.offset | Offset of the entry in the log file. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| log.syslog.facility.code | The Syslog numeric facility of the log event, if available. According to RFCs 5424 and 3164, this value should be an integer between 0 and 23. | long | -| log.syslog.priority | Syslog numeric priority of the event, if available. According to RFCs 5424 and 3164, the priority is 8 \* facility + severity. This number is therefore expected to contain a value between 0 and 191. | long | -| log.syslog.severity.code | The Syslog numeric severity of the log event, if available. If the event source publishing via Syslog provides a different numeric severity value (e.g. firewall, IDS), your source's numeric severity should go to `event.severity`. If the event source does not specify a distinct severity, you can optionally copy the Syslog severity to `event.severity`. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.application | When a specific application or service is identified from network connection details (source/dest IPs, ports, certificates, or wire format), this field captures the application's or service's name. For example, the original event identifies the network connection being from a specific web service in a `https` network connection, like `facebook` or `twitter`. The field value must be normalized to lowercase for querying. | keyword | -| network.bytes | Total bytes transferred in both directions. If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. | long | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.forwarded_ip | Host IP address when the source IP address is the proxy. | ip | -| network.name | Name given by operators to sections of their network. | keyword | -| network.packets | Total packets transferred in both directions. If `source.packets` and `destination.packets` are known, `network.packets` is their sum. | long | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| network.vlan.id | VLAN ID as reported by the observer. | keyword | -| observer.egress.interface.name | Interface name as reported by the system. | keyword | -| observer.hostname | Hostname of the observer. | keyword | -| observer.ingress.interface.name | Interface name as reported by the system. | keyword | -| observer.ingress.vlan.id | VLAN ID as reported by the observer. | keyword | -| observer.mac | MAC addresses of the observer. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| observer.product | The product name of the observer. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| observer.version | Observer version. | keyword | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.parent.name | Process name. Sometimes called program name or similar. | keyword | -| process.parent.name.text | Multi-field of `process.parent.name`. | match_only_text | -| process.parent.pid | Process id. | long | -| process.parent.title | Process title. The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. | keyword | -| process.parent.title.text | Multi-field of `process.parent.title`. | match_only_text | -| process.pid | Process id. | long | -| process.title | Process title. The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. | keyword | -| process.title.text | Multi-field of `process.title`. | match_only_text | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| server.domain | The domain name of the server system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| server.ip | IP address of the server (IPv4 or IPv6). | ip | -| server.mac | MAC address of the server. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| server.registered_domain | The highest registered server domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| server.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| server.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.mac | MAC address of the source. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| source.nat.ip | Translated ip of source based NAT sessions (e.g. internal client to internet) Typically connections traversing load balancers, firewalls, or routers. | ip | -| source.nat.port | Translated port of source based NAT sessions. (e.g. internal client to internet) Typically used with load balancers, firewalls, or routers. | long | -| source.port | Port of the source. | long | -| source.registered_domain | The highest registered source domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| source.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| source.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | An example event for `log` looks as following: @@ -435,175 +251,18 @@ An example event for `log` looks as following: | cisco_meraki.event.sentAt | Timestamp of the sent message (UTC) | date | | cisco_meraki.event.sharedSecret | User defined secret to be validated by the webhook receiver (optional) | keyword | | cisco_meraki.event.version | Current version of webhook format | keyword | -| client.domain | The domain name of the client system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| client.mac | MAC address of the client. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| client.registered_domain | The highest registered client domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| client.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| client.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.bytes | Bytes sent from the destination to the source. | long | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.mac | MAC address of the destination. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| destination.nat.ip | Translated ip of destination based NAT sessions (e.g. internet to private DMZ) Typically used with load balancers, firewalls, or routers. | ip | -| destination.nat.port | Port the source session is translated to by NAT Device. Typically used with load balancers, firewalls, or routers. | long | -| destination.port | Port of the destination. | long | -| destination.registered_domain | The highest registered destination domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| destination.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| destination.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| dns.answers.name | The domain name to which this resource record pertains. If a chain of CNAME is being resolved, each answer's `name` should be the one that corresponds with the answer's `data`. It should not simply be the original `question.name` repeated. | keyword | -| dns.answers.type | The type of data contained in this resource record. | keyword | -| dns.question.registered_domain | The highest registered domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| dns.question.subdomain | The subdomain is all of the labels under the registered_domain. If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| dns.question.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| dns.question.type | The type of record being queried. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.attributes | Array of file attributes. Attributes names will vary by platform. Here's a non-exhaustive list of values that are expected in this field: archive, compressed, directory, encrypted, execute, hidden, read, readonly, system, write. | keyword | -| file.directory | Directory where the file is located. It should include the drive letter, when appropriate. | keyword | -| file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| file.type | File type (file, dir, or symlink). | keyword | -| group.id | Unique identifier for the group on the system/platform. | keyword | -| group.name | Name of the group. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.request.referrer | Referrer for this HTTP request. | keyword | | input.type | Input type. | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | -| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | | log.offset | Offset of the entry in the log file. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| log.syslog.facility.code | The Syslog numeric facility of the log event, if available. According to RFCs 5424 and 3164, this value should be an integer between 0 and 23. | long | -| log.syslog.priority | Syslog numeric priority of the event, if available. According to RFCs 5424 and 3164, the priority is 8 \* facility + severity. This number is therefore expected to contain a value between 0 and 191. | long | -| log.syslog.severity.code | The Syslog numeric severity of the log event, if available. If the event source publishing via Syslog provides a different numeric severity value (e.g. firewall, IDS), your source's numeric severity should go to `event.severity`. If the event source does not specify a distinct severity, you can optionally copy the Syslog severity to `event.severity`. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.application | When a specific application or service is identified from network connection details (source/dest IPs, ports, certificates, or wire format), this field captures the application's or service's name. For example, the original event identifies the network connection being from a specific web service in a `https` network connection, like `facebook` or `twitter`. The field value must be normalized to lowercase for querying. | keyword | -| network.bytes | Total bytes transferred in both directions. If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. | long | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.forwarded_ip | Host IP address when the source IP address is the proxy. | ip | -| network.name | Name given by operators to sections of their network. | keyword | -| network.packets | Total packets transferred in both directions. If `source.packets` and `destination.packets` are known, `network.packets` is their sum. | long | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| network.vlan.id | VLAN ID as reported by the observer. | keyword | -| observer.egress.interface.name | Interface name as reported by the system. | keyword | -| observer.hostname | Hostname of the observer. | keyword | -| observer.ingress.interface.name | Interface name as reported by the system. | keyword | -| observer.mac | MAC addresses of the observer. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| observer.name | Custom name of the observer. This is a name that can be given to an observer. This can be helpful for example if multiple firewalls of the same model are used in an organization. If no custom name is needed, the field can be left empty. | keyword | -| observer.product | The product name of the observer. | keyword | -| observer.serial_number | Observer serial number. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| observer.version | Observer version. | keyword | -| organization.id | Unique identifier for the organization. | keyword | -| organization.name | Organization name. | keyword | -| organization.name.text | Multi-field of `organization.name`. | match_only_text | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.parent.name | Process name. Sometimes called program name or similar. | keyword | -| process.parent.name.text | Multi-field of `process.parent.name`. | match_only_text | -| process.parent.pid | Process id. | long | -| process.parent.title | Process title. The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. | keyword | -| process.parent.title.text | Multi-field of `process.parent.title`. | match_only_text | -| process.pid | Process id. | long | -| process.title | Process title. The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. | keyword | -| process.title.text | Multi-field of `process.title`. | match_only_text | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| server.domain | The domain name of the server system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| server.mac | MAC address of the server. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| server.registered_domain | The highest registered server domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| server.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| server.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.mac | MAC address of the source. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| source.nat.ip | Translated ip of source based NAT sessions (e.g. internal client to internet) Typically connections traversing load balancers, firewalls, or routers. | ip | -| source.nat.port | Translated port of source based NAT sessions. (e.g. internal client to internet) Typically used with load balancers, firewalls, or routers. | long | -| source.port | Port of the source. | long | -| source.registered_domain | The highest registered source domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| source.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| source.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| tags | List of keywords used to tag each event. | keyword | -| threat.indicator.description | Describes the type of action conducted by the threat. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.file.name | Name of the file including the extension, without the directory. | keyword | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.reference | Reference URL linking to additional information about this indicator. | keyword | -| threat.software.type | The type of software used by this threat to conduct behavior commonly modeled using MITRE ATT&CK®. While not required, you can use a MITRE ATT&CK® software type. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | An example event for `events` looks as following: diff --git a/packages/cisco_meraki/manifest.yml b/packages/cisco_meraki/manifest.yml index 49c669263b4..a90408378f1 100644 --- a/packages/cisco_meraki/manifest.yml +++ b/packages/cisco_meraki/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: cisco_meraki title: Cisco Meraki -version: "1.22.0" +version: "1.23.0" description: Collect logs from Cisco Meraki with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - security conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" screenshots: - src: /img/cisco-meraki-dashboard-1.png title: Cisco Meraki Dashboard From 754a9719b4c11db15f02fbae4d42bef70b63779e Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:42 +0930 Subject: [PATCH 025/121] [cisco_secure_endpoint] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/cisco_secure_endpoint --- packages/cisco_secure_endpoint/changelog.yml | 5 + .../data_stream/event/fields/agent.yml | 162 +----------------- .../data_stream/event/fields/base-fields.yml | 4 - .../data_stream/event/fields/ecs.yml | 118 ------------- packages/cisco_secure_endpoint/docs/README.md | 92 ---------- packages/cisco_secure_endpoint/manifest.yml | 4 +- 6 files changed, 8 insertions(+), 377 deletions(-) delete mode 100644 packages/cisco_secure_endpoint/data_stream/event/fields/ecs.yml diff --git a/packages/cisco_secure_endpoint/changelog.yml b/packages/cisco_secure_endpoint/changelog.yml index 417fbf5c7cf..82640b62fd2 100644 --- a/packages/cisco_secure_endpoint/changelog.yml +++ b/packages/cisco_secure_endpoint/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.26.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.25.0" changes: - description: Set sensitive values as secret. diff --git a/packages/cisco_secure_endpoint/data_stream/event/fields/agent.yml b/packages/cisco_secure_endpoint/data_stream/event/fields/agent.yml index 9dfc8d1aebc..2bc58530bac 100644 --- a/packages/cisco_secure_endpoint/data_stream/event/fields/agent.yml +++ b/packages/cisco_secure_endpoint/data_stream/event/fields/agent.yml @@ -5,175 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cisco_secure_endpoint/data_stream/event/fields/base-fields.yml b/packages/cisco_secure_endpoint/data_stream/event/fields/base-fields.yml index 351ac771303..7e2ae7c8427 100644 --- a/packages/cisco_secure_endpoint/data_stream/event/fields/base-fields.yml +++ b/packages/cisco_secure_endpoint/data_stream/event/fields/base-fields.yml @@ -18,10 +18,6 @@ type: constant_keyword description: Event dataset value: cisco_secure_endpoint.event -- name: container.id - description: Unique container id. - ignore_above: 1024 - type: keyword - name: input.type description: Type of Filebeat input. type: keyword diff --git a/packages/cisco_secure_endpoint/data_stream/event/fields/ecs.yml b/packages/cisco_secure_endpoint/data_stream/event/fields/ecs.yml deleted file mode 100644 index c8c2722750d..00000000000 --- a/packages/cisco_secure_endpoint/data_stream/event/fields/ecs.yml +++ /dev/null @@ -1,118 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: error.code -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.created -- external: ecs - name: event.code -- external: ecs - name: event.ingested -- name: event.kind - external: ecs -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.severity -- name: event.start - external: ecs -- external: ecs - name: event.category -- external: ecs - name: event.id -- external: ecs - name: event.timezone -- external: ecs - name: group.id -- name: related.ip - external: ecs -- name: related.user - external: ecs -- external: ecs - name: user.name -- external: ecs - name: user.domain -- external: ecs - name: user.email -- name: related.hosts - external: ecs -- name: related.hash - external: ecs -- name: process.args - external: ecs -- name: process.args_count - external: ecs -- name: process.command_line - external: ecs -- name: process.executable - external: ecs -- name: process.name - external: ecs -- name: process.pid - external: ecs -- name: process.hash.md5 - external: ecs -- name: process.hash.sha1 - external: ecs -- name: process.hash.sha256 - external: ecs -- name: file.hash.md5 - external: ecs -- name: file.hash.sha1 - external: ecs -- name: file.hash.sha256 - external: ecs -- name: file.name - external: ecs -- name: file.path - external: ecs -- external: ecs - name: destination.address -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.domain -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: network.direction -- external: ecs - name: network.transport -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: tags -- external: ecs - name: threat.tactic.id -- external: ecs - name: threat.tactic.reference -- external: ecs - name: threat.tactic.name -- external: ecs - name: threat.technique.id -- external: ecs - name: threat.technique.name -- external: ecs - name: threat.technique.reference diff --git a/packages/cisco_secure_endpoint/docs/README.md b/packages/cisco_secure_endpoint/docs/README.md index 63309531812..cd5420b08a2 100644 --- a/packages/cisco_secure_endpoint/docs/README.md +++ b/packages/cisco_secure_endpoint/docs/README.md @@ -186,106 +186,14 @@ An example event for `event` looks as following: | cisco.secure_endpoint.threat_hunting.techniques | List of all MITRE techniques related to the incident found. | flattened | | cisco.secure_endpoint.timestamp_nanoseconds | The timestamp in Epoch nanoseconds. | date | | cisco.secure_endpoint.vulnerabilities | An array of related vulnerabilities to the malicious event. | flattened | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.code | Error code describing the error. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| file.hash.md5 | MD5 hash. | keyword | -| file.hash.sha1 | SHA1 hash. | keyword | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| group.id | Unique identifier for the group on the system/platform. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| process.args | Array of process arguments, starting with the absolute path to the executable. May be filtered to protect sensitive information. | keyword | -| process.args_count | Length of the process.args array. This field can be useful for querying or performing bucket analysis on how many arguments were provided to start a process. More arguments may be an indication of suspicious activity. | long | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.executable | Absolute path to the process executable. | keyword | -| process.executable.text | Multi-field of `process.executable`. | match_only_text | -| process.hash.md5 | MD5 hash. | keyword | -| process.hash.sha1 | SHA1 hash. | keyword | -| process.hash.sha256 | SHA256 hash. | keyword | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.pid | Process id. | long | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| threat.tactic.id | The id of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/ ) | keyword | -| threat.tactic.name | Name of the type of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/) | keyword | -| threat.tactic.reference | The reference url of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/ ) | keyword | -| threat.technique.id | The id of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.name | The name of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.name.text | Multi-field of `threat.technique.name`. | match_only_text | -| threat.technique.reference | The reference url of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.email | User email address. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | diff --git a/packages/cisco_secure_endpoint/manifest.yml b/packages/cisco_secure_endpoint/manifest.yml index 01c4358f0cc..210f5d59c51 100644 --- a/packages/cisco_secure_endpoint/manifest.yml +++ b/packages/cisco_secure_endpoint/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: cisco_secure_endpoint title: Cisco Secure Endpoint -version: "2.25.0" +version: "2.26.0" description: Collect logs from Cisco Secure Endpoint (AMP) with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - edr_xdr conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" icons: - src: /img/cisco.svg title: cisco From 4589477e60f92284e44cf807df0f2b73591b76c6 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:44 +0930 Subject: [PATCH 026/121] [cisco_umbrella] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/cisco_umbrella --- packages/cisco_umbrella/changelog.yml | 5 + .../data_stream/log/fields/agent.yml | 139 +---------- .../data_stream/log/fields/base-fields.yml | 4 - .../data_stream/log/fields/ecs.yml | 216 ------------------ packages/cisco_umbrella/docs/README.md | 139 ----------- packages/cisco_umbrella/manifest.yml | 4 +- 6 files changed, 8 insertions(+), 499 deletions(-) diff --git a/packages/cisco_umbrella/changelog.yml b/packages/cisco_umbrella/changelog.yml index 330dc3bb1d1..2a1c95e0a65 100644 --- a/packages/cisco_umbrella/changelog.yml +++ b/packages/cisco_umbrella/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.25.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.24.1" changes: - description: Fix sample event. diff --git a/packages/cisco_umbrella/data_stream/log/fields/agent.yml b/packages/cisco_umbrella/data_stream/log/fields/agent.yml index 4d783629033..5e2d593b99e 100644 --- a/packages/cisco_umbrella/data_stream/log/fields/agent.yml +++ b/packages/cisco_umbrella/data_stream/log/fields/agent.yml @@ -5,152 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cisco_umbrella/data_stream/log/fields/base-fields.yml b/packages/cisco_umbrella/data_stream/log/fields/base-fields.yml index f3954559722..a15d8394c19 100644 --- a/packages/cisco_umbrella/data_stream/log/fields/base-fields.yml +++ b/packages/cisco_umbrella/data_stream/log/fields/base-fields.yml @@ -15,10 +15,6 @@ type: constant_keyword description: Event dataset value: cisco_umbrella.log -- name: container.id - description: Unique container id. - ignore_above: 1024 - type: keyword - name: input.type description: Type of Filebeat input. type: keyword diff --git a/packages/cisco_umbrella/data_stream/log/fields/ecs.yml b/packages/cisco_umbrella/data_stream/log/fields/ecs.yml index c2418fff333..44a60c5bd96 100644 --- a/packages/cisco_umbrella/data_stream/log/fields/ecs.yml +++ b/packages/cisco_umbrella/data_stream/log/fields/ecs.yml @@ -1,218 +1,2 @@ - external: ecs name: "@timestamp" -- external: ecs - name: client.domain -- external: ecs - name: client.registered_domain -- external: ecs - name: client.subdomain -- external: ecs - name: client.top_level_domain -- external: ecs - name: destination.address -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.bytes -- external: ecs - name: destination.domain -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.ip -- external: ecs - name: destination.mac -- external: ecs - name: destination.nat.ip -- external: ecs - name: destination.nat.port -- external: ecs - name: destination.port -- external: ecs - name: dns.response_code -- external: ecs - name: dns.question.type -- external: ecs - name: dns.type -- external: ecs - name: dns.question.name -- external: ecs - name: dns.question.registered_domain -- external: ecs - name: dns.question.subdomain -- external: ecs - name: dns.question.top_level_domain -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.code -- external: ecs - name: event.ingested -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.timezone -- external: ecs - name: file.name -- external: ecs - name: file.mime_type -- external: ecs - name: file.size -- external: ecs - name: file.hash.sha256 -- external: ecs - name: host.hostname -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: http.request.method -- external: ecs - name: http.request.mime_type -- external: ecs - name: http.request.referrer -- external: ecs - name: http.request.bytes -- external: ecs - name: http.response.status_code -- external: ecs - name: http.response.body.bytes -- external: ecs - name: http.response.bytes -- external: ecs - name: message -- external: ecs - name: network.application -- external: ecs - name: network.transport -- external: ecs - name: network.direction -- external: ecs - name: network.community_id -- external: ecs - name: network.name -- external: ecs - name: network.protocol -- external: ecs - name: observer.product -- external: ecs - name: observer.type -- external: ecs - name: observer.vendor -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: related.hash -- external: ecs - name: source.address -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.bytes -- external: ecs - name: source.domain -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: source.mac -- external: ecs - name: source.nat.ip -- external: ecs - name: source.nat.port -- external: ecs - name: source.port -- external: ecs - name: source.registered_domain -- external: ecs - name: source.subdomain -- external: ecs - name: source.top_level_domain -- external: ecs - name: tags -- external: ecs - name: url.domain -- external: ecs - name: url.original -- external: ecs - name: url.path -- external: ecs - name: url.query -- external: ecs - name: url.extension -- external: ecs - name: url.scheme -- external: ecs - name: url.full -- external: ecs - name: user.domain -- external: ecs - name: user.full_name -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user.email -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version -- external: ecs - name: user_agent.original -- external: ecs - name: rule.id -- external: ecs - name: rule.name -- external: ecs - name: log.file.path diff --git a/packages/cisco_umbrella/docs/README.md b/packages/cisco_umbrella/docs/README.md index 11f5de9a4a4..dfeebcdf629 100644 --- a/packages/cisco_umbrella/docs/README.md +++ b/packages/cisco_umbrella/docs/README.md @@ -177,153 +177,14 @@ An example event for `log` looks as following: | cisco.umbrella.sid | Used to uniquely identify signatures. | keyword | | cisco.umbrella.signature_list_id | Unique ID assigned to a Default or Custom Signature List. | keyword | | cisco.umbrella.warn_status | The warn page state associated with the request. | keyword | -| client.domain | The domain name of the client system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| client.registered_domain | The highest registered client domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| client.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| client.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.bytes | Bytes sent from the destination to the source. | long | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.mac | MAC address of the destination. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| destination.nat.ip | Translated ip of destination based NAT sessions (e.g. internet to private DMZ) Typically used with load balancers, firewalls, or routers. | ip | -| destination.nat.port | Port the source session is translated to by NAT Device. Typically used with load balancers, firewalls, or routers. | long | -| destination.port | Port of the destination. | long | -| dns.question.name | The name being queried. If the name field contains non-printable characters (below 32 or above 126), those characters should be represented as escaped base 10 integers (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, and line feeds should be converted to \t, \r, and \n respectively. | keyword | -| dns.question.registered_domain | The highest registered domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| dns.question.subdomain | The subdomain is all of the labels under the registered_domain. If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| dns.question.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| dns.question.type | The type of record being queried. | keyword | -| dns.response_code | The DNS response code. | keyword | -| dns.type | The type of DNS event captured, query or answer. If your source of DNS events only gives you DNS queries, you should only create dns events of type `dns.type:query`. If your source of DNS events gives you answers as well, you should create one event per query (optionally as soon as the query is seen). And a second event containing all query details as well as an array of answers. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.mime_type | MIME type should identify the format of the file or stream of bytes using https://www.iana.org/assignments/media-types/media-types.xhtml[IANA official types], where possible. When more than one type is applicable, the most specific type should be used. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.bytes | Total size in bytes of the request (body and headers). | long | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.request.mime_type | Mime type of the body of the request. This value must only be populated based on the content of the request body, not on the `Content-Type` header. Comparing the mime type of a request with the request's Content-Type header can be helpful in detecting threats or misconfigured clients. | keyword | -| http.request.referrer | Referrer for this HTTP request. | keyword | -| http.response.body.bytes | Size in bytes of the response body. | long | -| http.response.bytes | Total size in bytes of the response (body and headers). | long | -| http.response.status_code | HTTP response status code. | long | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.application | When a specific application or service is identified from network connection details (source/dest IPs, ports, certificates, or wire format), this field captures the application's or service's name. For example, the original event identifies the network connection being from a specific web service in a `https` network connection, like `facebook` or `twitter`. The field value must be normalized to lowercase for querying. | keyword | -| network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.name | Name given by operators to sections of their network. | keyword | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| observer.product | The product name of the observer. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.mac | MAC address of the source. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| source.nat.ip | Translated ip of source based NAT sessions (e.g. internal client to internet) Typically connections traversing load balancers, firewalls, or routers. | ip | -| source.nat.port | Translated port of source based NAT sessions. (e.g. internal client to internet) Typically used with load balancers, firewalls, or routers. | long | -| source.port | Port of the source. | long | -| source.registered_domain | The highest registered source domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| source.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| source.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| url.full.text | Multi-field of `url.full`. | match_only_text | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.email | User email address. | keyword | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | diff --git a/packages/cisco_umbrella/manifest.yml b/packages/cisco_umbrella/manifest.yml index d58aa633c5c..6edf48486d1 100644 --- a/packages/cisco_umbrella/manifest.yml +++ b/packages/cisco_umbrella/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: cisco_umbrella title: Cisco Umbrella -version: "1.24.1" +version: "1.25.0" description: Collect logs from Cisco Umbrella with Elastic Agent. type: integration categories: @@ -10,7 +10,7 @@ categories: - dns_security conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" icons: - src: /img/cisco.svg title: cisco From 7f68039da5075d70227c845f7c7234139fc74e74 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:47 +0930 Subject: [PATCH 027/121] [cloudflare] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/cloudflare --- packages/cloudflare/changelog.yml | 5 + .../data_stream/audit/fields/agent.yml | 50 ---- .../data_stream/audit/fields/beats.yml | 3 - .../data_stream/audit/fields/ecs.yml | 50 ---- .../data_stream/logpull/fields/agent.yml | 93 +------- .../data_stream/logpull/fields/beats.yml | 3 - .../data_stream/logpull/fields/ecs.yml | 222 ------------------ packages/cloudflare/docs/README.md | 190 --------------- packages/cloudflare/manifest.yml | 4 +- 9 files changed, 8 insertions(+), 612 deletions(-) delete mode 100644 packages/cloudflare/data_stream/audit/fields/ecs.yml delete mode 100644 packages/cloudflare/data_stream/logpull/fields/ecs.yml diff --git a/packages/cloudflare/changelog.yml b/packages/cloudflare/changelog.yml index 90778ddaba4..09cc681fc01 100644 --- a/packages/cloudflare/changelog.yml +++ b/packages/cloudflare/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.27.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.26.0" changes: - description: Improve handling of empty responses. diff --git a/packages/cloudflare/data_stream/audit/fields/agent.yml b/packages/cloudflare/data_stream/audit/fields/agent.yml index bca66ea4ae0..4b15225a4d4 100644 --- a/packages/cloudflare/data_stream/audit/fields/agent.yml +++ b/packages/cloudflare/data_stream/audit/fields/agent.yml @@ -1,56 +1,6 @@ -- name: cloud.account.id - external: ecs -- name: cloud.availability_zone - external: ecs -- name: cloud.instance.id - external: ecs -- name: cloud.instance.name - external: ecs -- name: cloud.machine.type - external: ecs -- name: cloud.provider - external: ecs -- name: cloud.region - external: ecs -- name: cloud.project.id - external: ecs - name: cloud.image.id type: keyword description: Image ID for the cloud instance. -- name: container.id - external: ecs -- name: container.image.name - external: ecs -- name: container.labels - external: ecs -- name: container.name - external: ecs -- name: host.architecture - external: ecs -- name: host.domain - external: ecs -- name: host.hostname - external: ecs -- name: host.id - external: ecs -- name: host.ip - external: ecs -- name: host.mac - external: ecs -- name: host.name - external: ecs -- name: host.os.family - external: ecs -- name: host.os.kernel - external: ecs -- name: host.os.name - external: ecs -- name: host.os.platform - external: ecs -- name: host.os.version - external: ecs -- name: host.type - external: ecs - name: host.containerized type: boolean description: If the host is a container. diff --git a/packages/cloudflare/data_stream/audit/fields/beats.yml b/packages/cloudflare/data_stream/audit/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/cloudflare/data_stream/audit/fields/beats.yml +++ b/packages/cloudflare/data_stream/audit/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/cloudflare/data_stream/audit/fields/ecs.yml b/packages/cloudflare/data_stream/audit/fields/ecs.yml deleted file mode 100644 index 2753c4e415e..00000000000 --- a/packages/cloudflare/data_stream/audit/fields/ecs.yml +++ /dev/null @@ -1,50 +0,0 @@ -- name: ecs.version - external: ecs -- name: error.message - external: ecs -- name: event.action - external: ecs -- name: event.id - external: ecs -- name: event.ingested - external: ecs -- name: event.created - external: ecs -- name: event.original - external: ecs -- name: event.outcome - external: ecs -- name: related.ip - external: ecs -- name: related.user - external: ecs -- name: source.address - external: ecs -- name: source.as.number - external: ecs -- name: source.as.organization.name - external: ecs -- name: source.geo.city_name - external: ecs -- name: source.geo.continent_name - external: ecs -- name: source.geo.country_iso_code - external: ecs -- name: source.geo.country_name - external: ecs -- name: source.geo.location - external: ecs -- name: source.geo.name - external: ecs -- name: source.geo.region_iso_code - external: ecs -- name: source.geo.region_name - external: ecs -- name: source.ip - external: ecs -- name: tags - external: ecs -- name: user.email - external: ecs -- name: user.id - external: ecs diff --git a/packages/cloudflare/data_stream/logpull/fields/agent.yml b/packages/cloudflare/data_stream/logpull/fields/agent.yml index 4d9a6f7b362..bc42d0a853b 100644 --- a/packages/cloudflare/data_stream/logpull/fields/agent.yml +++ b/packages/cloudflare/data_stream/logpull/fields/agent.yml @@ -1,100 +1,9 @@ - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cloudflare/data_stream/logpull/fields/beats.yml b/packages/cloudflare/data_stream/logpull/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/cloudflare/data_stream/logpull/fields/beats.yml +++ b/packages/cloudflare/data_stream/logpull/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/cloudflare/data_stream/logpull/fields/ecs.yml b/packages/cloudflare/data_stream/logpull/fields/ecs.yml deleted file mode 100644 index 84dce957ea0..00000000000 --- a/packages/cloudflare/data_stream/logpull/fields/ecs.yml +++ /dev/null @@ -1,222 +0,0 @@ -- name: client.as.number - external: ecs -- name: client.as.organization.name - external: ecs -- name: client.domain - external: ecs -- name: client.geo.city_name - external: ecs -- name: client.geo.country_name - external: ecs -- name: client.geo.country_iso_code - external: ecs -- name: client.geo.continent_name - external: ecs -- name: client.geo.region_iso_code - external: ecs -- name: client.geo.location - external: ecs -- name: client.geo.region_name - external: ecs -- name: client.ip - external: ecs -- name: client.address - external: ecs -- name: client.bytes - external: ecs -- name: client.port - external: ecs -- name: destination.bytes - external: ecs -- name: destination.as.number - external: ecs -- name: destination.as.organization.name - external: ecs -- name: destination.geo.city_name - external: ecs -- name: destination.geo.continent_name - external: ecs -- name: destination.geo.country_iso_code - external: ecs -- name: destination.geo.country_name - external: ecs -- name: destination.geo.location - external: ecs -- name: destination.geo.name - external: ecs -- name: destination.geo.region_iso_code - external: ecs -- name: destination.geo.region_name - external: ecs -- name: destination.ip - external: ecs -- name: destination.address - external: ecs -- name: ecs.version - external: ecs -- name: error.message - external: ecs -- name: event.action - external: ecs -- name: event.category - external: ecs -- name: event.id - external: ecs -- name: event.ingested - external: ecs -- name: event.created - external: ecs -- name: event.start - external: ecs -- name: event.end - external: ecs -- name: event.kind - external: ecs -- name: event.duration - external: ecs -- name: event.original - external: ecs -- name: event.outcome - external: ecs -- name: event.type - external: ecs -- name: message - external: ecs -- name: related.ip - external: ecs -- name: related.user - external: ecs -- name: source.address - external: ecs -- name: source.as.number - external: ecs -- name: source.as.organization.name - external: ecs -- name: source.bytes - external: ecs -- name: source.domain - external: ecs -- name: source.geo.city_name - external: ecs -- name: source.geo.continent_name - external: ecs -- name: source.geo.country_iso_code - external: ecs -- name: source.geo.country_name - external: ecs -- name: source.geo.location - external: ecs -- name: source.geo.name - external: ecs -- name: source.geo.region_iso_code - external: ecs -- name: source.geo.region_name - external: ecs -- name: source.ip - external: ecs -- name: source.port - external: ecs -- name: source.user.id - external: ecs -- name: source.user.full_name - external: ecs -- name: user_agent.device.name - external: ecs -- name: user_agent.name - external: ecs -- name: user_agent.original - external: ecs -- name: user_agent.os.name - external: ecs -- name: user_agent.os.version - external: ecs -- name: user_agent.os.full - external: ecs -- name: user_agent.version - external: ecs -- name: tags - external: ecs -- name: user.domain - external: ecs -- name: user.email - external: ecs -- name: user.id - external: ecs -- name: user.name - external: ecs -- name: user.full_name - external: ecs -- name: url.domain - external: ecs -- name: url.original - external: ecs -- name: url.password - external: ecs -- name: url.port - external: ecs -- name: url.username - external: ecs -- name: url.path - external: ecs -- name: url.query - external: ecs -- name: url.extension - external: ecs -- name: url.scheme - external: ecs -- name: url.full - external: ecs -- name: tls.cipher - external: ecs -- name: tls.version - external: ecs -- name: tls.version_protocol - external: ecs -- name: network.bytes - external: ecs -- name: network.protocol - external: ecs -- name: network.transport - external: ecs -- name: http.response.status_code - external: ecs -- name: http.request.body.bytes - external: ecs -- name: http.response.body.bytes - external: ecs -- name: http.request.method - external: ecs -- name: http.request.referrer - external: ecs -- name: http.version - external: ecs -- name: http.request.bytes - external: ecs -- name: http.response.bytes - external: ecs -- name: observer.type - external: ecs -- name: observer.vendor - external: ecs -- name: observer.geo.city_name - external: ecs -- name: observer.geo.continent_name - external: ecs -- name: observer.geo.country_iso_code - external: ecs -- name: observer.geo.country_name - external: ecs -- name: observer.geo.region_iso_code - external: ecs -- name: observer.geo.location - external: ecs -- name: observer.geo.region_name - external: ecs -- name: observer.ip - external: ecs -- name: server.address - external: ecs -- name: server.bytes - external: ecs -- name: server.ip - external: ecs diff --git a/packages/cloudflare/docs/README.md b/packages/cloudflare/docs/README.md index 49f81e22fa2..f30e25c925a 100644 --- a/packages/cloudflare/docs/README.md +++ b/packages/cloudflare/docs/README.md @@ -66,15 +66,7 @@ Audit logs summarize the history of changes made within your Cloudflare account. | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | | cloudflare.audit.actor.type | The type of actor, whether a User, Cloudflare Admin, or an Automated System. Valid values: user, admin, Cloudflare. | keyword | | cloudflare.audit.metadata | An object which can lend more context to the action being logged. This is a flexible value and varies between different actions. | flattened | | cloudflare.audit.new_value | The new value of the resource that was modified | flattened | @@ -82,62 +74,17 @@ Audit logs summarize the history of changes made within your Cloudflare account. | cloudflare.audit.owner.id | User identifier tag | keyword | | cloudflare.audit.resource.id | An identifier for the resource that was affected by the action | keyword | | cloudflare.audit.resource.type | A short string that describes the resource that was affected by the action | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | An example event for `audit` looks as following: @@ -239,21 +186,6 @@ These logs contain data related to the connecting client, the request path throu | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| client.address | Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| client.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| client.as.organization.name | Organization name. | keyword | -| client.as.organization.name.text | Multi-field of `client.as.organization.name`. | match_only_text | -| client.bytes | Bytes sent from the client to the server. | long | -| client.domain | The domain name of the client system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| client.geo.city_name | City name. | keyword | -| client.geo.continent_name | Name of the continent. | keyword | -| client.geo.country_iso_code | Country ISO code. | keyword | -| client.geo.country_name | Country name. | keyword | -| client.geo.location | Longitude and latitude. | geo_point | -| client.geo.region_iso_code | Region ISO code. | keyword | -| client.geo.region_name | Region name. | keyword | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| client.port | Port of the client. | long | | cloudflare.bot.score.src | Detection engine responsible for generating the Bot Score. Possible values are Not Computed, Heuristics, Machine Learning, Behavioral Analysis, Verified Bot, JS Fingerprinting, Cloudflare Service. | text | | cloudflare.bot.score.value | Cloudflare Bot Score. Scores below 30 are commonly associated with automated traffic. | long | | cloudflare.cache.bytes | Number of bytes returned by the cache | long | @@ -302,136 +234,14 @@ These logs contain data related to the connecting client, the request path throu | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.bytes | Bytes sent from the destination to the source. | long | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.duration | Duration of the event in nanoseconds. If `event.start` and `event.end` are known this value should be the difference between the end and start time. | long | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.body.bytes | Size in bytes of the request body. | long | -| http.request.bytes | Total size in bytes of the request (body and headers). | long | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.request.referrer | Referrer for this HTTP request. | keyword | -| http.response.body.bytes | Size in bytes of the response body. | long | -| http.response.bytes | Total size in bytes of the response (body and headers). | long | -| http.response.status_code | HTTP response status code. | long | -| http.version | HTTP version. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.bytes | Total bytes transferred in both directions. If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. | long | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| observer.geo.city_name | City name. | keyword | -| observer.geo.continent_name | Name of the continent. | keyword | -| observer.geo.country_iso_code | Country ISO code. | keyword | -| observer.geo.country_name | Country name. | keyword | -| observer.geo.location | Longitude and latitude. | geo_point | -| observer.geo.region_iso_code | Region ISO code. | keyword | -| observer.geo.region_name | Region name. | keyword | -| observer.ip | IP addresses of the observer. | ip | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| server.address | Some event server addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| server.bytes | Bytes sent from the server to the client. | long | -| server.ip | IP address of the server (IPv4 or IPv6). | ip | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| source.user.full_name | User's full name, if available. | keyword | -| source.user.full_name.text | Multi-field of `source.user.full_name`. | match_only_text | -| source.user.id | Unique identifier of the user. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| tls.cipher | String indicating the cipher used during the current connection. | keyword | -| tls.version | Numeric part of the version parsed from the original string. | keyword | -| tls.version_protocol | Normalized lowercase protocol name parsed from original string. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| url.full.text | Multi-field of `url.full`. | match_only_text | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.password | Password of the request. | keyword | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.username | Username of the request. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.email | User email address. | keyword | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | An example event for `logpull` looks as following: diff --git a/packages/cloudflare/manifest.yml b/packages/cloudflare/manifest.yml index 8152742eb73..71981c303fd 100644 --- a/packages/cloudflare/manifest.yml +++ b/packages/cloudflare/manifest.yml @@ -1,13 +1,13 @@ name: cloudflare title: Cloudflare -version: "2.26.0" +version: "2.27.0" description: Collect logs from Cloudflare with Elastic Agent. type: integration format_version: "3.0.2" categories: [security, network, cdn_security] conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" icons: - src: /img/cf-logo-v.svg title: Cloudflare From 2c385f37048a92ad1bd3553760a9f376e807d949 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:23:59 +0930 Subject: [PATCH 028/121] [cloudflare_logpush] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/cloudflare_logpush --- packages/cloudflare_logpush/changelog.yml | 5 + .../access_request/fields/agent.yml | 147 --- .../data_stream/access_request/fields/ecs.yml | 56 - .../data_stream/audit/fields/agent.yml | 147 --- .../data_stream/audit/fields/ecs.yml | 32 - .../data_stream/casb/fields/agent.yml | 147 --- .../data_stream/casb/fields/ecs.yml | 42 - .../device_posture/fields/agent.yml | 147 --- .../data_stream/device_posture/fields/ecs.yml | 32 - .../data_stream/dns/fields/agent.yml | 147 --- .../data_stream/dns/fields/ecs.yml | 22 - .../data_stream/dns_firewall/fields/agent.yml | 147 --- .../data_stream/dns_firewall/fields/ecs.yml | 52 - .../firewall_event/fields/agent.yml | 147 --- .../data_stream/firewall_event/fields/ecs.yml | 58 - .../data_stream/gateway_dns/fields/agent.yml | 147 --- .../data_stream/gateway_dns/fields/ecs.yml | 111 -- .../data_stream/gateway_http/fields/agent.yml | 147 --- .../data_stream/gateway_http/fields/ecs.yml | 124 -- .../gateway_network/fields/agent.yml | 147 --- .../gateway_network/fields/ecs.yml | 94 -- .../data_stream/http_request/fields/agent.yml | 147 --- .../data_stream/http_request/fields/ecs.yml | 64 - .../data_stream/magic_ids/fields/agent.yml | 147 --- .../data_stream/magic_ids/fields/ecs.yml | 82 -- .../data_stream/nel_report/fields/agent.yml | 147 --- .../data_stream/nel_report/fields/ecs.yml | 16 - .../network_analytics/fields/agent.yml | 147 --- .../network_analytics/fields/ecs.yml | 40 - .../network_session/fields/agent.yml | 147 --- .../network_session/fields/ecs.yml | 106 -- .../sinkhole_http/fields/agent.yml | 147 --- .../data_stream/sinkhole_http/fields/ecs.yml | 130 -- .../spectrum_event/fields/agent.yml | 147 --- .../data_stream/spectrum_event/fields/ecs.yml | 50 - .../workers_trace/fields/agent.yml | 147 --- .../data_stream/workers_trace/fields/ecs.yml | 48 - packages/cloudflare_logpush/docs/README.md | 1082 ----------------- packages/cloudflare_logpush/manifest.yml | 4 +- 39 files changed, 7 insertions(+), 4889 deletions(-) delete mode 100644 packages/cloudflare_logpush/data_stream/access_request/fields/ecs.yml delete mode 100644 packages/cloudflare_logpush/data_stream/audit/fields/ecs.yml delete mode 100644 packages/cloudflare_logpush/data_stream/casb/fields/ecs.yml delete mode 100644 packages/cloudflare_logpush/data_stream/device_posture/fields/ecs.yml delete mode 100644 packages/cloudflare_logpush/data_stream/dns/fields/ecs.yml delete mode 100644 packages/cloudflare_logpush/data_stream/dns_firewall/fields/ecs.yml delete mode 100644 packages/cloudflare_logpush/data_stream/firewall_event/fields/ecs.yml delete mode 100644 packages/cloudflare_logpush/data_stream/gateway_dns/fields/ecs.yml delete mode 100644 packages/cloudflare_logpush/data_stream/gateway_http/fields/ecs.yml delete mode 100644 packages/cloudflare_logpush/data_stream/gateway_network/fields/ecs.yml delete mode 100644 packages/cloudflare_logpush/data_stream/http_request/fields/ecs.yml delete mode 100644 packages/cloudflare_logpush/data_stream/magic_ids/fields/ecs.yml delete mode 100644 packages/cloudflare_logpush/data_stream/nel_report/fields/ecs.yml delete mode 100644 packages/cloudflare_logpush/data_stream/network_analytics/fields/ecs.yml delete mode 100644 packages/cloudflare_logpush/data_stream/network_session/fields/ecs.yml delete mode 100644 packages/cloudflare_logpush/data_stream/sinkhole_http/fields/ecs.yml delete mode 100644 packages/cloudflare_logpush/data_stream/spectrum_event/fields/ecs.yml delete mode 100644 packages/cloudflare_logpush/data_stream/workers_trace/fields/ecs.yml diff --git a/packages/cloudflare_logpush/changelog.yml b/packages/cloudflare_logpush/changelog.yml index 241a54bb14b..4b9deb70e05 100644 --- a/packages/cloudflare_logpush/changelog.yml +++ b/packages/cloudflare_logpush/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.21.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.20.0" changes: - description: Improve documentation on how to ingest data from Cloudflare R2. diff --git a/packages/cloudflare_logpush/data_stream/access_request/fields/agent.yml b/packages/cloudflare_logpush/data_stream/access_request/fields/agent.yml index 73e076a93b1..894e6f12be2 100644 --- a/packages/cloudflare_logpush/data_stream/access_request/fields/agent.yml +++ b/packages/cloudflare_logpush/data_stream/access_request/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cloudflare_logpush/data_stream/access_request/fields/ecs.yml b/packages/cloudflare_logpush/data_stream/access_request/fields/ecs.yml deleted file mode 100644 index e05613a828b..00000000000 --- a/packages/cloudflare_logpush/data_stream/access_request/fields/ecs.yml +++ /dev/null @@ -1,56 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: event.action -- external: ecs - name: event.id -- external: ecs - name: url.domain -- external: ecs - name: client.as.number -- external: ecs - name: client.as.organization.name -- external: ecs - name: client.geo.city_name -- external: ecs - name: client.geo.continent_code -- external: ecs - name: client.geo.continent_name -- external: ecs - name: client.geo.country_iso_code -- external: ecs - name: client.geo.country_name -- external: ecs - name: client.geo.location -- external: ecs - name: client.geo.name -- external: ecs - name: client.geo.postal_code -- external: ecs - name: client.geo.region_iso_code -- external: ecs - name: client.geo.region_name -- external: ecs - name: client.geo.timezone -- external: ecs - name: client.ip -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: user.id -- external: ecs - name: user.email -- external: ecs - name: tags diff --git a/packages/cloudflare_logpush/data_stream/audit/fields/agent.yml b/packages/cloudflare_logpush/data_stream/audit/fields/agent.yml index 73e076a93b1..894e6f12be2 100644 --- a/packages/cloudflare_logpush/data_stream/audit/fields/agent.yml +++ b/packages/cloudflare_logpush/data_stream/audit/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cloudflare_logpush/data_stream/audit/fields/ecs.yml b/packages/cloudflare_logpush/data_stream/audit/fields/ecs.yml deleted file mode 100644 index 625cce48d2d..00000000000 --- a/packages/cloudflare_logpush/data_stream/audit/fields/ecs.yml +++ /dev/null @@ -1,32 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.provider -- external: ecs - name: event.type -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: source.ip -- external: ecs - name: tags -- external: ecs - name: user.email -- external: ecs - name: user.id diff --git a/packages/cloudflare_logpush/data_stream/casb/fields/agent.yml b/packages/cloudflare_logpush/data_stream/casb/fields/agent.yml index 73e076a93b1..894e6f12be2 100644 --- a/packages/cloudflare_logpush/data_stream/casb/fields/agent.yml +++ b/packages/cloudflare_logpush/data_stream/casb/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cloudflare_logpush/data_stream/casb/fields/ecs.yml b/packages/cloudflare_logpush/data_stream/casb/fields/ecs.yml deleted file mode 100644 index af68abf1471..00000000000 --- a/packages/cloudflare_logpush/data_stream/casb/fields/ecs.yml +++ /dev/null @@ -1,42 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.id -- external: ecs - name: event.severity -- external: ecs - name: url.domain -- external: ecs - name: url.extension -- external: ecs - name: url.fragment -- external: ecs - name: url.full -- external: ecs - name: url.original -- external: ecs - name: url.password -- external: ecs - name: url.path -- external: ecs - name: url.port -- external: ecs - name: url.query -- external: ecs - name: url.registered_domain -- external: ecs - name: url.scheme -- external: ecs - name: url.subdomain -- external: ecs - name: url.top_level_domain -- external: ecs - name: url.username -- external: ecs - name: tags diff --git a/packages/cloudflare_logpush/data_stream/device_posture/fields/agent.yml b/packages/cloudflare_logpush/data_stream/device_posture/fields/agent.yml index 73e076a93b1..894e6f12be2 100644 --- a/packages/cloudflare_logpush/data_stream/device_posture/fields/agent.yml +++ b/packages/cloudflare_logpush/data_stream/device_posture/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cloudflare_logpush/data_stream/device_posture/fields/ecs.yml b/packages/cloudflare_logpush/data_stream/device_posture/fields/ecs.yml deleted file mode 100644 index 6d6d972a96e..00000000000 --- a/packages/cloudflare_logpush/data_stream/device_posture/fields/ecs.yml +++ /dev/null @@ -1,32 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: event.outcome -- external: ecs - name: user_agent.version -- external: ecs - name: rule.id -- external: ecs - name: rule.name -- external: ecs - name: rule.category -- external: ecs - name: related.hosts -- external: ecs - name: related.user -- external: ecs - name: user.id -- external: ecs - name: user.email -- external: ecs - name: tags diff --git a/packages/cloudflare_logpush/data_stream/dns/fields/agent.yml b/packages/cloudflare_logpush/data_stream/dns/fields/agent.yml index 73e076a93b1..894e6f12be2 100644 --- a/packages/cloudflare_logpush/data_stream/dns/fields/agent.yml +++ b/packages/cloudflare_logpush/data_stream/dns/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cloudflare_logpush/data_stream/dns/fields/ecs.yml b/packages/cloudflare_logpush/data_stream/dns/fields/ecs.yml deleted file mode 100644 index b1b0128da07..00000000000 --- a/packages/cloudflare_logpush/data_stream/dns/fields/ecs.yml +++ /dev/null @@ -1,22 +0,0 @@ -- external: ecs - name: dns.question.name -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: source.ip -- external: ecs - name: tags diff --git a/packages/cloudflare_logpush/data_stream/dns_firewall/fields/agent.yml b/packages/cloudflare_logpush/data_stream/dns_firewall/fields/agent.yml index 73e076a93b1..894e6f12be2 100644 --- a/packages/cloudflare_logpush/data_stream/dns_firewall/fields/agent.yml +++ b/packages/cloudflare_logpush/data_stream/dns_firewall/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cloudflare_logpush/data_stream/dns_firewall/fields/ecs.yml b/packages/cloudflare_logpush/data_stream/dns_firewall/fields/ecs.yml deleted file mode 100644 index 39240e89966..00000000000 --- a/packages/cloudflare_logpush/data_stream/dns_firewall/fields/ecs.yml +++ /dev/null @@ -1,52 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: event.timezone -- external: ecs - name: dns.question.name -- external: ecs - name: dns.response_code -- external: ecs - name: network.transport -- external: ecs - name: related.ip -- external: ecs - name: source.ip -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_code -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.name -- external: ecs - name: source.geo.postal_code -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.geo.timezone -- external: ecs - name: tags diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/fields/agent.yml b/packages/cloudflare_logpush/data_stream/firewall_event/fields/agent.yml index 73e076a93b1..894e6f12be2 100644 --- a/packages/cloudflare_logpush/data_stream/firewall_event/fields/agent.yml +++ b/packages/cloudflare_logpush/data_stream/firewall_event/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/fields/ecs.yml b/packages/cloudflare_logpush/data_stream/firewall_event/fields/ecs.yml deleted file mode 100644 index 18569146ce2..00000000000 --- a/packages/cloudflare_logpush/data_stream/firewall_event/fields/ecs.yml +++ /dev/null @@ -1,58 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: http.request.method -- external: ecs - name: http.response.status_code -- external: ecs - name: http.version -- external: ecs - name: network.protocol -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: rule.id -- external: ecs - name: source.as.number -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.ip -- external: ecs - name: tags -- external: ecs - name: url.domain -- external: ecs - name: url.path -- external: ecs - name: url.query -- external: ecs - name: url.scheme -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/fields/agent.yml b/packages/cloudflare_logpush/data_stream/gateway_dns/fields/agent.yml index 73e076a93b1..894e6f12be2 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_dns/fields/agent.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_dns/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/fields/ecs.yml b/packages/cloudflare_logpush/data_stream/gateway_dns/fields/ecs.yml deleted file mode 100644 index 5338b74884f..00000000000 --- a/packages/cloudflare_logpush/data_stream/gateway_dns/fields/ecs.yml +++ /dev/null @@ -1,111 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: event.outcome -- external: ecs - name: event.timezone -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_code -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.geo.name -- external: ecs - name: destination.geo.postal_code -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.geo.timezone -- external: ecs - name: dns.response_code -- external: ecs - name: dns.resolved_ip -- external: ecs - name: dns.question.name -- external: ecs - name: dns.question.type -- external: ecs - name: dns.answers - type: group -- external: ecs - name: dns.answers.class -- external: ecs - name: dns.answers.data -- external: ecs - name: dns.answers.name -- external: ecs - name: dns.answers.ttl -- external: ecs - name: dns.answers.type -- external: ecs - name: related.ip -- external: ecs - name: related.hosts -- external: ecs - name: related.user -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_code -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.name -- external: ecs - name: source.geo.postal_code -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.geo.timezone -- external: ecs - name: user.id -- external: ecs - name: user.email -- external: ecs - name: network.protocol -- external: ecs - name: tags diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/fields/agent.yml b/packages/cloudflare_logpush/data_stream/gateway_http/fields/agent.yml index 73e076a93b1..894e6f12be2 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_http/fields/agent.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_http/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/fields/ecs.yml b/packages/cloudflare_logpush/data_stream/gateway_http/fields/ecs.yml deleted file mode 100644 index e8fefa5175b..00000000000 --- a/packages/cloudflare_logpush/data_stream/gateway_http/fields/ecs.yml +++ /dev/null @@ -1,124 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: event.action -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_code -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.geo.name -- external: ecs - name: destination.geo.postal_code -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.geo.timezone -- external: ecs - name: http.request.method -- external: ecs - name: http.response.status_code -- external: ecs - name: http.version -- external: ecs - name: http.request.referrer -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_code -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.name -- external: ecs - name: source.geo.postal_code -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.geo.timezone -- external: ecs - name: url.domain -- external: ecs - name: url.extension -- external: ecs - name: url.fragment -- external: ecs - name: url.full -- external: ecs - name: url.original -- external: ecs - name: url.password -- external: ecs - name: url.path -- external: ecs - name: url.port -- external: ecs - name: url.query -- external: ecs - name: url.registered_domain -- external: ecs - name: url.scheme -- external: ecs - name: url.subdomain -- external: ecs - name: url.top_level_domain -- external: ecs - name: url.username -- external: ecs - name: user_agent.original -- external: ecs - name: user.id -- external: ecs - name: user.email -- external: ecs - name: related.ip -- external: ecs - name: related.hosts -- external: ecs - name: related.user -- external: ecs - name: tags diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/fields/agent.yml b/packages/cloudflare_logpush/data_stream/gateway_network/fields/agent.yml index 73e076a93b1..894e6f12be2 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_network/fields/agent.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_network/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/fields/ecs.yml b/packages/cloudflare_logpush/data_stream/gateway_network/fields/ecs.yml deleted file mode 100644 index 552aa9e4cdf..00000000000 --- a/packages/cloudflare_logpush/data_stream/gateway_network/fields/ecs.yml +++ /dev/null @@ -1,94 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: event.action -- external: ecs - name: event.id -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_code -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.geo.name -- external: ecs - name: destination.geo.postal_code -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.geo.timezone -- external: ecs - name: tls.client.server_name -- external: ecs - name: destination.domain -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_code -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.name -- external: ecs - name: source.geo.postal_code -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.geo.timezone -- external: ecs - name: network.transport -- external: ecs - name: user.id -- external: ecs - name: user.email -- external: ecs - name: related.ip -- external: ecs - name: related.hosts -- external: ecs - name: related.user -- external: ecs - name: tags diff --git a/packages/cloudflare_logpush/data_stream/http_request/fields/agent.yml b/packages/cloudflare_logpush/data_stream/http_request/fields/agent.yml index 73e076a93b1..894e6f12be2 100644 --- a/packages/cloudflare_logpush/data_stream/http_request/fields/agent.yml +++ b/packages/cloudflare_logpush/data_stream/http_request/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cloudflare_logpush/data_stream/http_request/fields/ecs.yml b/packages/cloudflare_logpush/data_stream/http_request/fields/ecs.yml deleted file mode 100644 index 1fc14e69fa0..00000000000 --- a/packages/cloudflare_logpush/data_stream/http_request/fields/ecs.yml +++ /dev/null @@ -1,64 +0,0 @@ -- external: ecs - name: destination.ip -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: http.request.method -- external: ecs - name: http.response.mime_type -- external: ecs - name: http.response.status_code -- external: ecs - name: http.version -- external: ecs - name: network.protocol -- external: ecs - name: related.hash -- external: ecs - name: related.ip -- external: ecs - name: source.as.number -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.ip -- external: ecs - name: tags -- external: ecs - name: tls.version -- external: ecs - name: tls.version_protocol -- external: ecs - name: url.domain -- external: ecs - name: url.original -- external: ecs - name: url.path -- external: ecs - name: url.query -- external: ecs - name: url.scheme -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version diff --git a/packages/cloudflare_logpush/data_stream/magic_ids/fields/agent.yml b/packages/cloudflare_logpush/data_stream/magic_ids/fields/agent.yml index 73e076a93b1..894e6f12be2 100644 --- a/packages/cloudflare_logpush/data_stream/magic_ids/fields/agent.yml +++ b/packages/cloudflare_logpush/data_stream/magic_ids/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cloudflare_logpush/data_stream/magic_ids/fields/ecs.yml b/packages/cloudflare_logpush/data_stream/magic_ids/fields/ecs.yml deleted file mode 100644 index e262f4f6bc3..00000000000 --- a/packages/cloudflare_logpush/data_stream/magic_ids/fields/ecs.yml +++ /dev/null @@ -1,82 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: event.action -- external: ecs - name: event.timezone -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_code -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.geo.name -- external: ecs - name: destination.geo.postal_code -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.geo.timezone -- external: ecs - name: related.ip -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_code -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.name -- external: ecs - name: source.geo.postal_code -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.geo.timezone -- external: ecs - name: network.transport -- external: ecs - name: tags diff --git a/packages/cloudflare_logpush/data_stream/nel_report/fields/agent.yml b/packages/cloudflare_logpush/data_stream/nel_report/fields/agent.yml index 73e076a93b1..894e6f12be2 100644 --- a/packages/cloudflare_logpush/data_stream/nel_report/fields/agent.yml +++ b/packages/cloudflare_logpush/data_stream/nel_report/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cloudflare_logpush/data_stream/nel_report/fields/ecs.yml b/packages/cloudflare_logpush/data_stream/nel_report/fields/ecs.yml deleted file mode 100644 index a0c9092f429..00000000000 --- a/packages/cloudflare_logpush/data_stream/nel_report/fields/ecs.yml +++ /dev/null @@ -1,16 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: error.type -- external: ecs - name: tags diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/fields/agent.yml b/packages/cloudflare_logpush/data_stream/network_analytics/fields/agent.yml index 73e076a93b1..894e6f12be2 100644 --- a/packages/cloudflare_logpush/data_stream/network_analytics/fields/agent.yml +++ b/packages/cloudflare_logpush/data_stream/network_analytics/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cloudflare_logpush/data_stream/network_analytics/fields/ecs.yml b/packages/cloudflare_logpush/data_stream/network_analytics/fields/ecs.yml deleted file mode 100644 index eb6b3e0ad5c..00000000000 --- a/packages/cloudflare_logpush/data_stream/network_analytics/fields/ecs.yml +++ /dev/null @@ -1,40 +0,0 @@ -- external: ecs - name: destination.as.number -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.type -- external: ecs - name: network.community_id -- external: ecs - name: network.direction -- external: ecs - name: network.transport -- external: ecs - name: related.hash -- external: ecs - name: related.ip -- external: ecs - name: rule.id -- external: ecs - name: source.as.number -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: tags diff --git a/packages/cloudflare_logpush/data_stream/network_session/fields/agent.yml b/packages/cloudflare_logpush/data_stream/network_session/fields/agent.yml index 73e076a93b1..894e6f12be2 100644 --- a/packages/cloudflare_logpush/data_stream/network_session/fields/agent.yml +++ b/packages/cloudflare_logpush/data_stream/network_session/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cloudflare_logpush/data_stream/network_session/fields/ecs.yml b/packages/cloudflare_logpush/data_stream/network_session/fields/ecs.yml deleted file mode 100644 index 85bba103ac0..00000000000 --- a/packages/cloudflare_logpush/data_stream/network_session/fields/ecs.yml +++ /dev/null @@ -1,106 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: event.action -- external: ecs - name: event.id -- external: ecs - name: event.start -- external: ecs - name: event.end -- external: ecs - name: device.id -- external: ecs - name: device.model.identifier -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: destination.bytes -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_code -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.geo.name -- external: ecs - name: destination.geo.postal_code -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.geo.timezone -- external: ecs - name: tls.server.issuer -- external: ecs - name: network.transport -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_code -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.name -- external: ecs - name: source.geo.postal_code -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.geo.timezone -- external: ecs - name: source.bytes -- external: ecs - name: user.id -- external: ecs - name: user.email -- external: ecs - name: network.vlan.id -- external: ecs - name: related.ip -- external: ecs - name: related.hosts -- external: ecs - name: related.user -- external: ecs - name: tags diff --git a/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/agent.yml b/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/agent.yml index 73e076a93b1..894e6f12be2 100644 --- a/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/agent.yml +++ b/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/ecs.yml b/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/ecs.yml deleted file mode 100644 index 438064b2091..00000000000 --- a/packages/cloudflare_logpush/data_stream/sinkhole_http/fields/ecs.yml +++ /dev/null @@ -1,130 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: event.timezone -- external: ecs - name: destination.ip -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_code -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.geo.name -- external: ecs - name: destination.geo.postal_code -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.geo.timezone -- external: ecs - name: http.request.body.content -- external: ecs - name: http.request.body.bytes -- external: ecs - name: http.request.method -- external: ecs - name: http.request.referrer -- external: ecs - name: related.ip -- external: ecs - name: related.hosts -- external: ecs - name: related.user -- external: ecs - name: source.ip -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_code -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.name -- external: ecs - name: source.geo.postal_code -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.geo.timezone -- external: ecs - name: url.domain -- external: ecs - name: url.extension -- external: ecs - name: url.fragment -- external: ecs - name: url.full -- external: ecs - name: url.original -- external: ecs - name: url.password -- external: ecs - name: url.path -- external: ecs - name: url.port -- external: ecs - name: url.query -- external: ecs - name: url.registered_domain -- external: ecs - name: url.scheme -- external: ecs - name: url.subdomain -- external: ecs - name: url.top_level_domain -- external: ecs - name: url.username -- external: ecs - name: user.name -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version -- external: ecs - name: tags diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/fields/agent.yml b/packages/cloudflare_logpush/data_stream/spectrum_event/fields/agent.yml index 73e076a93b1..894e6f12be2 100644 --- a/packages/cloudflare_logpush/data_stream/spectrum_event/fields/agent.yml +++ b/packages/cloudflare_logpush/data_stream/spectrum_event/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cloudflare_logpush/data_stream/spectrum_event/fields/ecs.yml b/packages/cloudflare_logpush/data_stream/spectrum_event/fields/ecs.yml deleted file mode 100644 index fea89d47ec6..00000000000 --- a/packages/cloudflare_logpush/data_stream/spectrum_event/fields/ecs.yml +++ /dev/null @@ -1,50 +0,0 @@ -- external: ecs - name: destination.bytes -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: ecs.version -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.end -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.start -- external: ecs - name: event.type -- external: ecs - name: http.response.status_code -- external: ecs - name: network.community_id -- external: ecs - name: network.transport -- external: ecs - name: related.ip -- external: ecs - name: source.as.number -- external: ecs - name: source.bytes -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: tags -- external: ecs - name: tls.version -- external: ecs - name: tls.version_protocol diff --git a/packages/cloudflare_logpush/data_stream/workers_trace/fields/agent.yml b/packages/cloudflare_logpush/data_stream/workers_trace/fields/agent.yml index 73e076a93b1..894e6f12be2 100644 --- a/packages/cloudflare_logpush/data_stream/workers_trace/fields/agent.yml +++ b/packages/cloudflare_logpush/data_stream/workers_trace/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/cloudflare_logpush/data_stream/workers_trace/fields/ecs.yml b/packages/cloudflare_logpush/data_stream/workers_trace/fields/ecs.yml deleted file mode 100644 index 8993427eec6..00000000000 --- a/packages/cloudflare_logpush/data_stream/workers_trace/fields/ecs.yml +++ /dev/null @@ -1,48 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: http.request.method -- external: ecs - name: http.response.status_code -- external: ecs - name: tags -- external: ecs - name: url.domain -- external: ecs - name: url.extension -- external: ecs - name: url.fragment -- external: ecs - name: url.full -- external: ecs - name: url.original -- external: ecs - name: url.password -- external: ecs - name: url.path -- external: ecs - name: url.port -- external: ecs - name: url.query -- external: ecs - name: url.registered_domain -- external: ecs - name: url.scheme -- external: ecs - name: url.subdomain -- external: ecs - name: url.top_level_domain -- external: ecs - name: url.username diff --git a/packages/cloudflare_logpush/docs/README.md b/packages/cloudflare_logpush/docs/README.md index eb7f4c7b41e..5acdba78f8b 100644 --- a/packages/cloudflare_logpush/docs/README.md +++ b/packages/cloudflare_logpush/docs/README.md @@ -297,30 +297,7 @@ An example event for `access_request` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| client.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| client.as.organization.name | Organization name. | keyword | -| client.as.organization.name.text | Multi-field of `client.as.organization.name`. | match_only_text | -| client.geo.city_name | City name. | keyword | -| client.geo.continent_code | Two-letter code representing continent's name. | keyword | -| client.geo.continent_name | Name of the continent. | keyword | -| client.geo.country_iso_code | Country ISO code. | keyword | -| client.geo.country_name | Country name. | keyword | -| client.geo.location | Longitude and latitude. | geo_point | -| client.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| client.geo.postal_code | Postal code associated with the location. Values appropriate for this field may also be known as a postcode or ZIP code and will vary widely from country to country. | keyword | -| client.geo.region_iso_code | Region ISO code. | keyword | -| client.geo.region_name | Region name. | keyword | -| client.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | | cloudflare_logpush.access_request.action | What type of record is this. login | logout. | keyword | | cloudflare_logpush.access_request.allowed | If request was allowed or denied. | boolean | | cloudflare_logpush.access_request.app.domain | The domain of the Application that Access is protecting. | keyword | @@ -336,49 +313,17 @@ An example event for `access_request` looks as following: | cloudflare_logpush.access_request.timestamp | The date and time the corresponding access request was made. | date | | cloudflare_logpush.access_request.user.email | Email of the user who logged in. | keyword | | cloudflare_logpush.access_request.user.id | The uid of the user who logged in. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | ### audit @@ -499,15 +444,7 @@ An example event for `audit` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | | cloudflare_logpush.audit.action.result | Whether the action was successful. | keyword | | cloudflare_logpush.audit.action.type | Type of action taken. | keyword | | cloudflare_logpush.audit.actor.email | Email of the actor. | keyword | @@ -523,51 +460,17 @@ An example event for `audit` looks as following: | cloudflare_logpush.audit.resource.id | Unique identifier of the resource within Cloudflare system. | keyword | | cloudflare_logpush.audit.resource.type | The type of resource that was changed. | keyword | | cloudflare_logpush.audit.timestamp | When the change happened. | date | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | ### casb @@ -712,15 +615,7 @@ An example event for `casb` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | | cloudflare_logpush.casb.asset.id | Unique identifier for an asset of this type. Format will vary by policy vendor. | keyword | | cloudflare_logpush.casb.asset.metadata | Metadata associated with the asset. Structure will vary by policy vendor. | flattened | | cloudflare_logpush.casb.asset.name | Asset display name. | keyword | @@ -733,58 +628,17 @@ An example event for `casb` looks as following: | cloudflare_logpush.casb.integration.name | Human-readable name of the integration. | keyword | | cloudflare_logpush.casb.integration.policy_vendor | Human-readable vendor name of the integration´s policy. | keyword | | cloudflare_logpush.casb.timestamp | Date and time the finding was first identified. | date | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | -| url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| url.full.text | Multi-field of `url.full`. | match_only_text | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.password | Password of the request. | keyword | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| url.username | Username of the request. | keyword | ### device_posture @@ -921,15 +775,7 @@ An example event for `device_posture` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | | cloudflare_logpush.device_posture.eval.expected | JSON object of what the posture check expects from the Zero Trust client. | flattened | | cloudflare_logpush.device_posture.eval.received | JSON object of what the Zero Trust client actually uploads. | flattened | | cloudflare_logpush.device_posture.eval.result | Whether this posture upload passes the associated posture check, given the requirements posture check at the time of the timestamp. | boolean | @@ -947,51 +793,17 @@ An example event for `device_posture` looks as following: | cloudflare_logpush.device_posture.user.email | The email used to register the device with the Zero Trust client. | keyword | | cloudflare_logpush.device_posture.user.id | The uid of the user who registered the device. | keyword | | cloudflare_logpush.device_posture.version | The Zero Trust client version at the time of upload. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.category | A categorization value keyword used by the entity using the rule for detection of this event. | keyword | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user_agent.version | Version of the user agent. | keyword | ### dns @@ -1093,15 +905,7 @@ An example event for `dns` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | | cloudflare_logpush.dns.colo.code | IATA airport code of data center that received the request. | keyword | | cloudflare_logpush.dns.edns.subnet | EDNS Client Subnet (IPv4 or IPv6). | ip | | cloudflare_logpush.dns.edns.subnet_length | EDNS Client Subnet length. | long | @@ -1111,46 +915,17 @@ An example event for `dns` looks as following: | cloudflare_logpush.dns.response.code | Integer value of response code. | long | | cloudflare_logpush.dns.source.ip | IP address of the client (IPv4 or IPv6). | ip | | cloudflare_logpush.dns.timestamp | Timestamp at which the query occurred. | date | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| dns.question.name | The name being queried. If the name field contains non-printable characters (below 32 or above 126), those characters should be represented as escaped base 10 integers (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, and line feeds should be converted to \t, \r, and \n respectively. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | ### dns_firewall @@ -1280,15 +1055,7 @@ An example event for `dns_firewall` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | | cloudflare_logpush.dns_firewall.cluster_id | The ID of the cluster which handled this request. | keyword | | cloudflare_logpush.dns_firewall.colo.code | IATA airport code of data center that received the request. | keyword | | cloudflare_logpush.dns_firewall.edns.subnet | EDNS Client Subnet (IPv4 or IPv6). | ip | @@ -1307,62 +1074,17 @@ An example event for `dns_firewall` looks as following: | cloudflare_logpush.dns_firewall.upstream.ip | IP of the upstream nameserver (IPv4 or IPv6). | ip | | cloudflare_logpush.dns_firewall.upstream.response_code | Response code from the upstream nameserver. | keyword | | cloudflare_logpush.dns_firewall.upstream.response_time_ms | Upstream response time in milliseconds. | long | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| dns.question.name | The name being queried. If the name field contains non-printable characters (below 32 or above 126), those characters should be represented as escaped base 10 integers (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, and line feeds should be converted to \t, \r, and \n respectively. | keyword | -| dns.response_code | The DNS response code. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_code | Two-letter code representing continent's name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.postal_code | Postal code associated with the location. Values appropriate for this field may also be known as a postcode or ZIP code and will vary widely from country to country. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | ### firewall_event @@ -1541,15 +1263,7 @@ An example event for `firewall_event` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | | cloudflare_logpush.firewall_event.action | The code of the first-class action the Cloudflare Firewall took on this request. | keyword | | cloudflare_logpush.firewall_event.client.asn.description | The ASN of the visitor as string. | keyword | | cloudflare_logpush.firewall_event.client.asn.value | The ASN number of the visitor. | long | @@ -1578,67 +1292,17 @@ An example event for `firewall_event` looks as following: | cloudflare_logpush.firewall_event.rule.id | The Cloudflare security product-specific RuleID triggered by this request. | keyword | | cloudflare_logpush.firewall_event.source | The Cloudflare security product triggered by this request. | keyword | | cloudflare_logpush.firewall_event.timestamp | The date and time the event occurred at the edge. | date | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.response.status_code | HTTP response status code. | long | -| http.version | HTTP version. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.path | Path of the request, such as "/search". | wildcard | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | ### gateway_dns @@ -1881,15 +1545,7 @@ An example event for `gateway_dns` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | | cloudflare_logpush.gateway_dns.answers | The response data objects. | flattened | | cloudflare_logpush.gateway_dns.application_id | ID of the application the domain belongs to. | long | | cloudflare_logpush.gateway_dns.colo.code | The name of the colo that received the DNS query . | keyword | @@ -1922,92 +1578,17 @@ An example event for `gateway_dns` looks as following: | cloudflare_logpush.gateway_dns.timezone_inferred_method | Method used to pick the time zone for the schedule. | keyword | | cloudflare_logpush.gateway_dns.user.email | Email used to authenticate the client. | keyword | | cloudflare_logpush.gateway_dns.user.id | User identity where the HTTP request originated from. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_code | Two-letter code representing continent's name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| destination.geo.postal_code | Postal code associated with the location. Values appropriate for this field may also be known as a postcode or ZIP code and will vary widely from country to country. | keyword | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| dns.answers | An array containing an object for each answer section returned by the server. The main keys that should be present in these objects are defined by ECS. Records that have more information may contain more keys than what ECS defines. Not all DNS data sources give all details about DNS answers. At minimum, answer objects must contain the `data` key. If more information is available, map as much of it to ECS as possible, and add any additional fields to the answer objects as custom fields. | group | -| dns.answers.class | The class of DNS data contained in this resource record. | keyword | -| dns.answers.data | The data describing the resource. The meaning of this data depends on the type and class of the resource record. | keyword | -| dns.answers.name | The domain name to which this resource record pertains. If a chain of CNAME is being resolved, each answer's `name` should be the one that corresponds with the answer's `data`. It should not simply be the original `question.name` repeated. | keyword | -| dns.answers.ttl | The time interval in seconds that this resource record may be cached before it should be discarded. Zero values mean that the data should not be cached. | long | -| dns.answers.type | The type of data contained in this resource record. | keyword | -| dns.question.name | The name being queried. If the name field contains non-printable characters (below 32 or above 126), those characters should be represented as escaped base 10 integers (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, and line feeds should be converted to \t, \r, and \n respectively. | keyword | -| dns.question.type | The type of record being queried. | keyword | -| dns.resolved_ip | Array containing all IPs seen in `answers.data`. The `answers` array can be difficult to use, because of the variety of data formats it can contain. Extracting all IP addresses seen in there to `dns.resolved_ip` makes it possible to index them as IP addresses, and makes them easier to visualize and query for. | ip | -| dns.response_code | The DNS response code. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_code | Two-letter code representing continent's name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.postal_code | Postal code associated with the location. Values appropriate for this field may also be known as a postcode or ZIP code and will vary widely from country to country. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | ### gateway_http @@ -2221,15 +1802,7 @@ An example event for `gateway_http` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | | cloudflare_logpush.gateway_http.account_id | Cloudflare account tag. | keyword | | cloudflare_logpush.gateway_http.action | Action performed by gateway on the HTTP request. | keyword | | cloudflare_logpush.gateway_http.blocked_file.hash | Hash of the file blocked in the response, if any. | keyword | @@ -2262,102 +1835,17 @@ An example event for `gateway_http` looks as following: | cloudflare_logpush.gateway_http.user.email | Email used to authenticate the client. | keyword | | cloudflare_logpush.gateway_http.user.id | User identity where the HTTP request originated from. | keyword | | cloudflare_logpush.gateway_http.user_agent | Contents of the user agent header in the HTTP request. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_code | Two-letter code representing continent's name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| destination.geo.postal_code | Postal code associated with the location. Values appropriate for this field may also be known as a postcode or ZIP code and will vary widely from country to country. | keyword | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.request.referrer | Referrer for this HTTP request. | keyword | -| http.response.status_code | HTTP response status code. | long | -| http.version | HTTP version. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_code | Two-letter code representing continent's name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.postal_code | Postal code associated with the location. Values appropriate for this field may also be known as a postcode or ZIP code and will vary widely from country to country. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | -| url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| url.full.text | Multi-field of `url.full`. | match_only_text | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.password | Password of the request. | keyword | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| url.username | Username of the request. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | ### gateway_network @@ -2530,15 +2018,7 @@ An example event for `gateway_network` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | | cloudflare_logpush.gateway_network.account_id | Cloudflare account tag. | keyword | | cloudflare_logpush.gateway_network.action | Action performed by gateway on the session. | keyword | | cloudflare_logpush.gateway_network.destination.ip | Destination IP of the network session. | ip | @@ -2558,84 +2038,17 @@ An example event for `gateway_network` looks as following: | cloudflare_logpush.gateway_network.transport | Transport protocol used for this session. | keyword | | cloudflare_logpush.gateway_network.user.email | Email associated with the user identity where the network sesion originated from. | keyword | | cloudflare_logpush.gateway_network.user.id | User identity where the network session originated from. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_code | Two-letter code representing continent's name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| destination.geo.postal_code | Postal code associated with the location. Values appropriate for this field may also be known as a postcode or ZIP code and will vary widely from country to country. | keyword | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_code | Two-letter code representing continent's name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.postal_code | Postal code associated with the location. Values appropriate for this field may also be known as a postcode or ZIP code and will vary widely from country to country. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| tls.client.server_name | Also called an SNI, this tells the server which hostname to which the client is attempting to connect to. When this value is available, it should get copied to `destination.domain`. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | ### http_request @@ -2935,15 +2348,7 @@ An example event for `http_request` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | | cloudflare_logpush.http_request.bot.detection_ids | List of IDs that correlate to the Bot Management Heuristic detections made on a request. Available in Logpush v2 only. | long | | cloudflare_logpush.http_request.bot.score.src | Detection engine responsible for generating the Bot Score. Possible values are Not Computed, Heuristics, Machine Learning, Behavioral Analysis, Verified Bot, JS Fingerprinting, Cloudflare Service. | text | | cloudflare_logpush.http_request.bot.score.value | Cloudflare Bot Score. Scores below 30 are commonly associated with automated traffic. | long | @@ -3034,71 +2439,17 @@ An example event for `http_request` looks as following: | cloudflare_logpush.http_request.worker.wall_time_us | Real-time in microseconds elapsed between start and end of worker invocation. | long | | cloudflare_logpush.http_request.zone.id | Internal zone ID. | long | | cloudflare_logpush.http_request.zone.name | The human-readable name of the zone. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.response.mime_type | Mime type of the body of the response. This value must only be populated based on the content of the response body, not on the `Content-Type` header. Comparing the mime type of a response with the response's Content-Type header can be helpful in detecting misconfigured servers. | keyword | -| http.response.status_code | HTTP response status code. | long | -| http.version | HTTP version. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| tls.version | Numeric part of the version parsed from the original string. | keyword | -| tls.version_protocol | Normalized lowercase protocol name parsed from original string. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | ### magic_ids @@ -3237,15 +2588,7 @@ An example event for `magic_ids` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | | cloudflare_logpush.magic_ids.action | What action was taken on the packet. Possible values are pass | block. | keyword | | cloudflare_logpush.magic_ids.colo.city | The city where the detection occurred. | keyword | | cloudflare_logpush.magic_ids.colo.code | The IATA airport code corresponding to where the detection occurred. | keyword | @@ -3258,78 +2601,17 @@ An example event for `magic_ids` looks as following: | cloudflare_logpush.magic_ids.source.port | The source port of the packet which triggered the detection. It is set to 0 if the protocol field is set to any. | long | | cloudflare_logpush.magic_ids.timestamp | A timestamp of when the detection occurred. | date | | cloudflare_logpush.magic_ids.transport | The layer 4 protocol of the packet which triggered the detection. Possible values are tcp | udp | any. Variant any means a detection occurred at a lower layer (such as IP). | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_code | Two-letter code representing continent's name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| destination.geo.postal_code | Postal code associated with the location. Values appropriate for this field may also be known as a postcode or ZIP code and will vary widely from country to country. | keyword | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_code | Two-letter code representing continent's name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.postal_code | Postal code associated with the location. Values appropriate for this field may also be known as a postcode or ZIP code and will vary widely from country to country. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | ### nel_report @@ -3420,15 +2702,7 @@ An example event for `nel_report` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | | cloudflare_logpush.nel_report.client.ip.asn.description | Client ASN description. | keyword | | cloudflare_logpush.nel_report.client.ip.asn.value | Client ASN. | long | | cloudflare_logpush.nel_report.client.ip.country | Client country. | keyword | @@ -3436,43 +2710,17 @@ An example event for `nel_report` looks as following: | cloudflare_logpush.nel_report.last_known_good.colo.code | IATA airport code of colo client connected to. | keyword | | cloudflare_logpush.nel_report.phase | The phase of connection the error occurred in. | keyword | | cloudflare_logpush.nel_report.timestamp | Timestamp for error report. | date | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.type | The type of the error, for example the class name of the exception. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | List of keywords used to tag each event. | keyword | ### network_analytics @@ -3725,15 +2973,7 @@ An example event for `network_analytics` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | | cloudflare_logpush.network_analytics.attack.campaign.id | Unique identifier of the attack campaign that this packet was a part of, if any. | keyword | | cloudflare_logpush.network_analytics.attack.id | Unique identifier of the mitigation that matched the packet, if any. | keyword | | cloudflare_logpush.network_analytics.colo.country | The country of colo that received the packet (ISO 3166-1 alpha-2). | keyword | @@ -3815,55 +3055,17 @@ An example event for `network_analytics` looks as following: | cloudflare_logpush.network_analytics.udp.checksum | Value of the Checksum header field in the UDP packet. | long | | cloudflare_logpush.network_analytics.udp.payload_length | Value of the Payload Length header field in the UDP packet. | long | | cloudflare_logpush.network_analytics.verdict | The action that Cloudflare systems think should be taken on the packet (pass | drop). | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | ### network_session @@ -4082,15 +3284,7 @@ An example event for `network_session` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | | cloudflare_logpush.network_session.account_id | Cloudflare account ID. | keyword | | cloudflare_logpush.network_session.destination.bytes | The number of bytes sent from the origin to the client during the network session. | long | | cloudflare_logpush.network_session.destination.ip | The IP of the destination (origin) for the network session. | ip | @@ -4129,90 +3323,17 @@ An example event for `network_session` looks as following: | cloudflare_logpush.network_session.user.email | Email address associated with the user identity which initiated the network session. | keyword | | cloudflare_logpush.network_session.user.id | User identity where the network session originated from. | keyword | | cloudflare_logpush.network_session.vlan.id | Identifier of the virtual network configured for the client. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.bytes | Bytes sent from the destination to the source. | long | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_code | Two-letter code representing continent's name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| destination.geo.postal_code | Postal code associated with the location. Values appropriate for this field may also be known as a postcode or ZIP code and will vary widely from country to country. | keyword | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| device.id | The unique identifier of a device. The identifier must not change across application sessions but stay fixed for an instance of a (mobile) device. On iOS, this value must be equal to the vendor identifier (https://developer.apple.com/documentation/uikit/uidevice/1620059-identifierforvendor). On Android, this value must be equal to the Firebase Installation ID or a globally unique UUID which is persisted across sessions in your application. For GDPR and data protection law reasons this identifier should not carry information that would allow to identify a user. | keyword | -| device.model.identifier | The machine readable identifier of the device model. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| network.vlan.id | VLAN ID as reported by the observer. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_code | Two-letter code representing continent's name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.postal_code | Postal code associated with the location. Values appropriate for this field may also be known as a postcode or ZIP code and will vary widely from country to country. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| tls.server.issuer | Subject of the issuer of the x.509 certificate presented by the server. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | ### sinkhole_http @@ -4391,15 +3512,7 @@ An example event for `sinkhole_http` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | | cloudflare_logpush.sinkhole_http.account_id | The Account ID. | keyword | | cloudflare_logpush.sinkhole_http.destination.ip | The destination IP address of the request. | ip | | cloudflare_logpush.sinkhole_http.host.name | The host the request was sent to. | keyword | @@ -4417,109 +3530,17 @@ An example event for `sinkhole_http` looks as following: | cloudflare_logpush.sinkhole_http.timestamp | The date and time the sinkhole HTTP request was logged. | date | | cloudflare_logpush.sinkhole_http.user.name | The request username. | keyword | | cloudflare_logpush.sinkhole_http.user_agent | The request user agent. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_code | Two-letter code representing continent's name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| destination.geo.postal_code | Postal code associated with the location. Values appropriate for this field may also be known as a postcode or ZIP code and will vary widely from country to country. | keyword | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.body.bytes | Size in bytes of the request body. | long | -| http.request.body.content | The full HTTP request body. | wildcard | -| http.request.body.content.text | Multi-field of `http.request.body.content`. | match_only_text | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.request.referrer | Referrer for this HTTP request. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_code | Two-letter code representing continent's name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.postal_code | Postal code associated with the location. Values appropriate for this field may also be known as a postcode or ZIP code and will vary widely from country to country. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | -| url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| url.full.text | Multi-field of `url.full`. | match_only_text | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.password | Password of the request. | keyword | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| url.username | Username of the request. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | ### spectrum_event @@ -4670,15 +3691,7 @@ An example event for `spectrum_event` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | | cloudflare_logpush.spectrum_event.action | Event Action. | keyword | | cloudflare_logpush.spectrum_event.application | The unique public ID of the application on which the event occurred. | keyword | | cloudflare_logpush.spectrum_event.client.asn | Client AS number. | long | @@ -4710,60 +3723,17 @@ An example event for `spectrum_event` looks as following: | cloudflare_logpush.spectrum_event.proxy.protocol | Which form of proxy protocol is applied to the given connection. | keyword | | cloudflare_logpush.spectrum_event.status | A code indicating reason for connection closure. | long | | cloudflare_logpush.spectrum_event.timestamp | Timestamp at which the event took place. | date | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.bytes | Bytes sent from the destination to the source. | long | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.response.status_code | HTTP response status code. | long | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| tls.version | Numeric part of the version parsed from the original string. | keyword | -| tls.version_protocol | Normalized lowercase protocol name parsed from original string. | keyword | ### workers_trace @@ -4887,15 +3857,7 @@ An example event for `workers_trace` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | | cloudflare_logpush.workers_trace.dispatch_namespace | The Cloudflare Worker dispatch namespace. | keyword | | cloudflare_logpush.workers_trace.event | Details about the source event. | flattened | | cloudflare_logpush.workers_trace.exceptions | List of uncaught exceptions during the invocation. | flattened | @@ -4905,59 +3867,15 @@ An example event for `workers_trace` looks as following: | cloudflare_logpush.workers_trace.script.tags | A list of user-defined tags used to categorize the Worker. | keyword | | cloudflare_logpush.workers_trace.timestamp | The timestamp of when the event was received. | date | | cloudflare_logpush.workers_trace.type | The event type that triggered the invocation. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.response.status_code | HTTP response status code. | long | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | -| url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| url.full.text | Multi-field of `url.full`. | match_only_text | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.password | Password of the request. | keyword | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| url.username | Username of the request. | keyword | diff --git a/packages/cloudflare_logpush/manifest.yml b/packages/cloudflare_logpush/manifest.yml index 23767102f73..5297d180aeb 100644 --- a/packages/cloudflare_logpush/manifest.yml +++ b/packages/cloudflare_logpush/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: cloudflare_logpush title: Cloudflare Logpush -version: "1.20.0" +version: "1.21.0" description: Collect and parse logs from Cloudflare API with Elastic Agent. type: integration categories: @@ -10,7 +10,7 @@ categories: - cdn_security conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" screenshots: - src: /img/cloudflare_logpush-overview1.png title: Cloudflare Logpush - Zero Trust Overview From 56a71ba457e25cddf1549d4bb668638ff1289158 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:24:00 +0930 Subject: [PATCH 029/121] [cribl] - change to ECS version git@v8.11.0 ECS version in build manifest changed from git@8.11 to git@v8.11.0. Removed import_mappings. The set ecs.version processor in pipelines was changed 8.11.0. Previously the pipeline was setting version 8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/cribl --- packages/cribl/_dev/build/build.yml | 3 +-- packages/cribl/changelog.yml | 5 +++++ .../logs/elasticsearch/ingest_pipeline/default.yml | 2 +- packages/cribl/data_stream/logs/fields/ecs.yml | 4 ---- packages/cribl/manifest.yml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) delete mode 100644 packages/cribl/data_stream/logs/fields/ecs.yml diff --git a/packages/cribl/_dev/build/build.yml b/packages/cribl/_dev/build/build.yml index bc1ffa5e1eb..2bfcfc223b0 100644 --- a/packages/cribl/_dev/build/build.yml +++ b/packages/cribl/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: - reference: git@8.11 - import_mappings: true + reference: "git@v8.11.0" diff --git a/packages/cribl/changelog.yml b/packages/cribl/changelog.yml index cb5384708cb..1881e0e85d7 100644 --- a/packages/cribl/changelog.yml +++ b/packages/cribl/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.4.0" + changes: + - description: ECS version updated to 8.11.0. Removed import_mappings. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "0.3.0" changes: - description: Update manifest format version to v3.0.3. diff --git a/packages/cribl/data_stream/logs/elasticsearch/ingest_pipeline/default.yml b/packages/cribl/data_stream/logs/elasticsearch/ingest_pipeline/default.yml index c1b465abebf..e1378d7d99f 100644 --- a/packages/cribl/data_stream/logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cribl/data_stream/logs/elasticsearch/ingest_pipeline/default.yml @@ -3,7 +3,7 @@ description: Pipeline for rerouting log streams from Cribl. processors: - set: field: ecs.version - value: 8.13.0 + value: 8.11.0 - append: field: tags value: diff --git a/packages/cribl/data_stream/logs/fields/ecs.yml b/packages/cribl/data_stream/logs/fields/ecs.yml deleted file mode 100644 index 74989720ba2..00000000000 --- a/packages/cribl/data_stream/logs/fields/ecs.yml +++ /dev/null @@ -1,4 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: tags diff --git a/packages/cribl/manifest.yml b/packages/cribl/manifest.yml index 66cb85b6031..b2b280b1242 100644 --- a/packages/cribl/manifest.yml +++ b/packages/cribl/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.3 name: cribl title: "Cribl" -version: 0.3.0 +version: "0.4.0" description: Stream logs from Cribl into Elastic. type: integration categories: From 74040766a0595d4f2a163773331df1c1e8825529 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:24:09 +0930 Subject: [PATCH 030/121] [crowdstrike] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/crowdstrike --- packages/crowdstrike/_dev/build/build.yml | 1 - packages/crowdstrike/changelog.yml | 5 + .../data_stream/alert/fields/beats.yml | 3 - .../data_stream/falcon/fields/agent.yml | 159 +---------- .../data_stream/falcon/fields/beats.yml | 2 - .../data_stream/falcon/fields/ecs.yml | 166 ------------ .../data_stream/fdr/fields/ecs.yml | 242 ----------------- .../data_stream/host/fields/beats.yml | 3 - packages/crowdstrike/docs/README.md | 254 ------------------ packages/crowdstrike/manifest.yml | 4 +- 10 files changed, 8 insertions(+), 831 deletions(-) delete mode 100644 packages/crowdstrike/data_stream/falcon/fields/ecs.yml delete mode 100644 packages/crowdstrike/data_stream/fdr/fields/ecs.yml diff --git a/packages/crowdstrike/_dev/build/build.yml b/packages/crowdstrike/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/crowdstrike/_dev/build/build.yml +++ b/packages/crowdstrike/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/crowdstrike/changelog.yml b/packages/crowdstrike/changelog.yml index 496c6041ef3..1332187ceda 100644 --- a/packages/crowdstrike/changelog.yml +++ b/packages/crowdstrike/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.37.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.36.0" changes: - description: Add `device.id` field. diff --git a/packages/crowdstrike/data_stream/alert/fields/beats.yml b/packages/crowdstrike/data_stream/alert/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/crowdstrike/data_stream/alert/fields/beats.yml +++ b/packages/crowdstrike/data_stream/alert/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/crowdstrike/data_stream/falcon/fields/agent.yml b/packages/crowdstrike/data_stream/falcon/fields/agent.yml index 388ddad84cd..2bc58530bac 100644 --- a/packages/crowdstrike/data_stream/falcon/fields/agent.yml +++ b/packages/crowdstrike/data_stream/falcon/fields/agent.yml @@ -5,172 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - external: ecs - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - external: ecs - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/crowdstrike/data_stream/falcon/fields/beats.yml b/packages/crowdstrike/data_stream/falcon/fields/beats.yml index b13d5cc96f4..96190255552 100644 --- a/packages/crowdstrike/data_stream/falcon/fields/beats.yml +++ b/packages/crowdstrike/data_stream/falcon/fields/beats.yml @@ -7,5 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - external: ecs diff --git a/packages/crowdstrike/data_stream/falcon/fields/ecs.yml b/packages/crowdstrike/data_stream/falcon/fields/ecs.yml deleted file mode 100644 index bb9fb2639b4..00000000000 --- a/packages/crowdstrike/data_stream/falcon/fields/ecs.yml +++ /dev/null @@ -1,166 +0,0 @@ -- name: message - external: ecs -- name: ecs.version - external: ecs -- name: event.code - external: ecs -- name: event.kind - external: ecs -- name: event.category - external: ecs -- name: event.type - external: ecs -- name: event.action - external: ecs -- name: event.original - external: ecs -- name: event.ingested - external: ecs -- name: event.created - external: ecs -- name: event.outcome - external: ecs -- name: event.url - external: ecs -- name: event.severity - external: ecs -- name: event.start - external: ecs -- name: event.end - external: ecs -- name: user.id - external: ecs -- name: user.name - external: ecs -- name: user.domain - external: ecs -- name: user.email - external: ecs -- name: threat.technique.name - external: ecs -- name: threat.technique.id - external: ecs -- name: threat.tactic.name - external: ecs -- name: threat.tactic.id - external: ecs -- name: threat.framework - external: ecs -- name: process.pid - external: ecs -- name: process.start - external: ecs -- name: process.end - external: ecs -- name: process.name - external: ecs -- name: process.command_line - external: ecs -- name: process.args - external: ecs -- name: process.executable - external: ecs -- name: process.parent.executable - external: ecs -- name: process.parent.pid - external: ecs -- name: process.parent.command_line - external: ecs -- name: process.parent.args - external: ecs -- name: device.id - external: ecs -- name: agent.name - external: ecs -- name: agent.id - external: ecs -- name: agent.type - external: ecs -- name: agent.version - external: ecs -- name: source.ip - external: ecs -- name: source.port - external: ecs -- name: destination.ip - external: ecs -- name: destination.port - external: ecs -- name: file.hash.sha1 - external: ecs -- name: file.hash.sha256 - external: ecs -- name: file.hash.md5 - external: ecs -- name: file.path - external: ecs -- name: rule.author - external: ecs -- name: rule.id - external: ecs -- name: rule.uuid - external: ecs -- name: rule.name - external: ecs -- name: rule.description - external: ecs -- name: error.message - external: ecs -- name: rule.ruleset - external: ecs -- name: rule.category - external: ecs -- name: network.direction - external: ecs -- name: network.type - external: ecs -- name: related.ip - external: ecs -- name: related.user - external: ecs -- name: related.hosts - external: ecs -- name: related.hash - external: ecs -- name: tags - external: ecs -- name: observer.vendor - external: ecs -- name: observer.product - external: ecs -- name: source.as.number - external: ecs -- name: source.as.organization.name - external: ecs -- name: source.geo.city_name - external: ecs -- name: source.geo.continent_name - external: ecs -- name: source.geo.country_iso_code - external: ecs -- name: source.geo.country_name - external: ecs -- name: source.geo.location - external: ecs -- name: source.geo.region_iso_code - external: ecs -- name: source.geo.region_name - external: ecs -- name: destination.as.number - external: ecs -- name: destination.as.organization.name - external: ecs -- name: destination.geo.city_name - external: ecs -- name: destination.geo.continent_name - external: ecs -- name: destination.geo.country_iso_code - external: ecs -- name: destination.geo.country_name - external: ecs -- name: destination.geo.location - external: ecs -- name: destination.geo.region_iso_code - external: ecs -- name: destination.geo.region_name - external: ecs diff --git a/packages/crowdstrike/data_stream/fdr/fields/ecs.yml b/packages/crowdstrike/data_stream/fdr/fields/ecs.yml deleted file mode 100644 index a59ddff9346..00000000000 --- a/packages/crowdstrike/data_stream/fdr/fields/ecs.yml +++ /dev/null @@ -1,242 +0,0 @@ -- external: ecs - name: destination.address -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: device.id -- external: ecs - name: dns.question.name -- external: ecs - name: dns.question.registered_domain -- external: ecs - name: dns.question.subdomain -- external: ecs - name: dns.question.top_level_domain -- external: ecs - name: dns.question.type -- external: ecs - name: dns.type -- external: ecs - name: ecs.version -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.timezone -- external: ecs - name: event.type -- external: ecs - name: file.device -- external: ecs - name: file.directory -- external: ecs - name: file.drive_letter -- external: ecs - name: file.extension -- external: ecs - name: file.hash.sha256 -- external: ecs - name: file.inode -- external: ecs - name: file.name -- external: ecs - name: file.path -- external: ecs - name: file.size -- external: ecs - name: file.type -- external: ecs - name: host.geo.city_name -- external: ecs - name: host.geo.continent_name -- external: ecs - name: host.geo.country_name -- external: ecs - name: host.geo.timezone -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.ip -- external: ecs - name: host.name -- external: ecs - name: log.file.path -- external: ecs - name: network.community_id -- external: ecs - name: network.direction -- external: ecs - name: network.iana_number -- external: ecs - name: network.transport -- external: ecs - name: observer.geo.city_name -- external: ecs - name: observer.geo.continent_name -- external: ecs - name: observer.geo.country_iso_code -- external: ecs - name: observer.geo.country_name -- external: ecs - name: observer.geo.location -- external: ecs - name: observer.geo.region_iso_code -- external: ecs - name: observer.geo.region_name -- external: ecs - name: observer.ip -- external: ecs - name: observer.serial_number -- external: ecs - name: observer.type -- external: ecs - name: observer.vendor -- external: ecs - name: observer.version -- external: ecs - name: host.os.type -- external: ecs - name: host.os.version -- external: ecs - name: process.args -- external: ecs - name: process.args_count -- external: ecs - name: process.command_line -- external: ecs - name: process.end -- external: ecs - name: process.entity_id -- external: ecs - name: process.executable -- external: ecs - name: process.name -- external: ecs - name: process.exit_code -- external: ecs - name: process.hash.md5 -- external: ecs - name: process.hash.sha256 -- external: ecs - name: process.parent.entity_id -- external: ecs - name: process.parent.name -- external: ecs - name: process.pgid -- external: ecs - name: process.pid -- external: ecs - name: process.start -- external: ecs - name: process.thread.id -- external: ecs - name: process.title -- external: ecs - name: process.uptime -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: server.address -- external: ecs - name: server.domain -- external: ecs - name: server.registered_domain -- external: ecs - name: server.subdomain -- external: ecs - name: server.top_level_domain -- external: ecs - name: source.address -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: source.mac -- external: ecs - name: source.port -- external: ecs - name: tags -- external: ecs - name: url.domain -- external: ecs - name: url.extension -- external: ecs - name: url.original -- external: ecs - name: url.path -- external: ecs - name: url.registered_domain -- external: ecs - name: url.scheme -- external: ecs - name: url.subdomain -- external: ecs - name: url.top_level_domain -- external: ecs - name: user.domain -- external: ecs - name: user.email -- external: ecs - name: user.full_name -- external: ecs - name: user.group.id -- external: ecs - name: user.id -- external: ecs - name: user.name diff --git a/packages/crowdstrike/data_stream/host/fields/beats.yml b/packages/crowdstrike/data_stream/host/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/crowdstrike/data_stream/host/fields/beats.yml +++ b/packages/crowdstrike/data_stream/host/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/crowdstrike/docs/README.md b/packages/crowdstrike/docs/README.md index d3817dd18a7..fbb790c0e2a 100644 --- a/packages/crowdstrike/docs/README.md +++ b/packages/crowdstrike/docs/README.md @@ -549,7 +549,6 @@ An example event for `alert` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Falcon @@ -584,23 +583,7 @@ Current supported event types are: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| agent.id | Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id. | keyword | -| agent.name | Custom name of the agent. This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. | keyword | -| agent.type | Type of the agent. The agent type always stays the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine. | keyword | -| agent.version | Version of the agent. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | crowdstrike.event.AccountCreationTimeStamp | The timestamp of when the source account was created in Active Directory. | date | | crowdstrike.event.AccountId | | keyword | | crowdstrike.event.ActivityId | ID of the activity that triggered the detection. | keyword | @@ -818,118 +801,14 @@ Current supported event types are: | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| device.id | The unique identifier of a device. The identifier must not change across application sessions but stay fixed for an instance of a (mobile) device. On iOS, this value must be equal to the vendor identifier (https://developer.apple.com/documentation/uikit/uidevice/1620059-identifierforvendor). On Android, this value must be equal to the Firebase Installation ID or a globally unique UUID which is persisted across sessions in your application. For GDPR and data protection law reasons this identifier should not carry information that would allow to identify a user. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| event.url | URL linking to an external system to continue investigation of this event. This URL links to another system where in-depth investigation of the specific occurrence of this event can take place. Alert events, indicated by `event.kind:alert`, are a common use case for this field. | keyword | -| file.hash.md5 | MD5 hash. | keyword | -| file.hash.sha1 | SHA1 hash. | keyword | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | -| observer.product | The product name of the observer. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| process.args | Array of process arguments, starting with the absolute path to the executable. May be filtered to protect sensitive information. | keyword | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.end | The time the process ended. | date | -| process.executable | Absolute path to the process executable. | keyword | -| process.executable.text | Multi-field of `process.executable`. | match_only_text | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.parent.args | Array of process arguments, starting with the absolute path to the executable. May be filtered to protect sensitive information. | keyword | -| process.parent.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.parent.command_line.text | Multi-field of `process.parent.command_line`. | match_only_text | -| process.parent.executable | Absolute path to the process executable. | keyword | -| process.parent.executable.text | Multi-field of `process.parent.executable`. | match_only_text | -| process.parent.pid | Process id. | long | -| process.pid | Process id. | long | -| process.start | The time the process started. | date | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.author | Name, organization, or pseudonym of the author or authors who created the rule used to generate this event. | keyword | -| rule.category | A categorization value keyword used by the entity using the rule for detection of this event. | keyword | -| rule.description | The description of the rule generating the event. | keyword | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| rule.ruleset | Name of the ruleset, policy, group, or parent category in which the rule used to generate this event is a member. | keyword | -| rule.uuid | A rule ID that is unique within the scope of a set or group of agents, observers, or other entities using the rule for detection of this event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| threat.framework | Name of the threat framework used to further categorize and classify the tactic and technique of the reported threat. Framework classification can be provided by detecting systems, evaluated at ingest time, or retrospectively tagged to events. | keyword | -| threat.tactic.id | The id of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/ ) | keyword | -| threat.tactic.name | Name of the type of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/) | keyword | -| threat.technique.id | The id of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.name | The name of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.name.text | Multi-field of `threat.technique.name`. | match_only_text | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | An example event for `falcon` looks as following: @@ -1560,143 +1439,11 @@ and/or `session_token`. | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| device.id | The unique identifier of a device. The identifier must not change across application sessions but stay fixed for an instance of a (mobile) device. On iOS, this value must be equal to the vendor identifier (https://developer.apple.com/documentation/uikit/uidevice/1620059-identifierforvendor). On Android, this value must be equal to the Firebase Installation ID or a globally unique UUID which is persisted across sessions in your application. For GDPR and data protection law reasons this identifier should not carry information that would allow to identify a user. | keyword | -| dns.question.name | The name being queried. If the name field contains non-printable characters (below 32 or above 126), those characters should be represented as escaped base 10 integers (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, and line feeds should be converted to \t, \r, and \n respectively. | keyword | -| dns.question.registered_domain | The highest registered domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| dns.question.subdomain | The subdomain is all of the labels under the registered_domain. If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| dns.question.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| dns.question.type | The type of record being queried. | keyword | -| dns.type | The type of DNS event captured, query or answer. If your source of DNS events only gives you DNS queries, you should only create dns events of type `dns.type:query`. If your source of DNS events gives you answers as well, you should create one event per query (optionally as soon as the query is seen). And a second event containing all query details as well as an array of answers. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.device | Device that is the source of the file. | keyword | -| file.directory | Directory where the file is located. It should include the drive letter, when appropriate. | keyword | -| file.drive_letter | Drive letter where the file is located. This field is only relevant on Windows. The value should be uppercase, and not include the colon. | keyword | -| file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.inode | Inode representing the file in the filesystem. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| file.type | File type (file, dir, or symlink). | keyword | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.geo.city_name | City name. | keyword | -| host.geo.continent_name | Name of the continent. | keyword | -| host.geo.country_name | Country name. | keyword | -| host.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.ip | Host ip addresses. | ip | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | -| host.os.type | Use the `os.type` field to categorize the operating system into one of the broad commercial families. If the OS you're dealing with is not listed as an expected value, the field should not be populated. Please let us know by opening an issue with ECS, to propose its addition. | keyword | -| host.os.version | Operating system version as a raw string. | keyword | | input.type | | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | | long | -| network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | | observer.address | | keyword | -| observer.geo.city_name | City name. | keyword | -| observer.geo.continent_name | Name of the continent. | keyword | -| observer.geo.country_iso_code | Country ISO code. | keyword | -| observer.geo.country_name | Country name. | keyword | -| observer.geo.location | Longitude and latitude. | geo_point | -| observer.geo.region_iso_code | Region ISO code. | keyword | -| observer.geo.region_name | Region name. | keyword | -| observer.ip | IP addresses of the observer. | ip | -| observer.serial_number | Observer serial number. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| observer.version | Observer version. | keyword | -| process.args | Array of process arguments, starting with the absolute path to the executable. May be filtered to protect sensitive information. | keyword | -| process.args_count | Length of the process.args array. This field can be useful for querying or performing bucket analysis on how many arguments were provided to start a process. More arguments may be an indication of suspicious activity. | long | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.end | The time the process ended. | date | -| process.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.executable | Absolute path to the process executable. | keyword | -| process.executable.text | Multi-field of `process.executable`. | match_only_text | -| process.exit_code | The exit code of the process, if this is a termination event. The field should be absent if there is no exit code for the event (e.g. process start). | long | -| process.hash.md5 | MD5 hash. | keyword | -| process.hash.sha256 | SHA256 hash. | keyword | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.parent.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.parent.name | Process name. Sometimes called program name or similar. | keyword | -| process.parent.name.text | Multi-field of `process.parent.name`. | match_only_text | -| process.pgid | Deprecated for removal in next major version release. This field is superseded by `process.group_leader.pid`. Identifier of the group of processes the process belongs to. | long | -| process.pid | Process id. | long | -| process.start | The time the process started. | date | -| process.thread.id | Thread ID. | long | -| process.title | Process title. The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. | keyword | -| process.title.text | Multi-field of `process.title`. | match_only_text | -| process.uptime | Seconds the process has been up. | long | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| server.address | Some event server addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| server.domain | The domain name of the server system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| server.registered_domain | The highest registered server domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| server.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| server.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.mac | MAC address of the source. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.email | User email address. | keyword | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.group.id | Unique identifier for the group on the system/platform. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | An example event for `fdr` looks as following: @@ -2246,5 +1993,4 @@ An example event for `host` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | diff --git a/packages/crowdstrike/manifest.yml b/packages/crowdstrike/manifest.yml index 1a056eeec20..d3c9bf33f15 100644 --- a/packages/crowdstrike/manifest.yml +++ b/packages/crowdstrike/manifest.yml @@ -1,13 +1,13 @@ name: crowdstrike title: CrowdStrike -version: "1.36.0" +version: "1.37.0" description: Collect logs from Crowdstrike with Elastic Agent. type: integration format_version: "3.0.3" categories: [security, edr_xdr] conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" icons: - src: /img/logo-integrations-crowdstrike.svg title: CrowdStrike From d2c3bc6a0c04f4364c5155f6e93c455b20b111c3 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:24:17 +0930 Subject: [PATCH 031/121] [cyberarkpas] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.7.1 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/cyberarkpas --- packages/cyberarkpas/changelog.yml | 5 + ...lear-users-history-start.log-expected.json | 2 +- ...-clear-users-history-end.log-expected.json | 2 +- ...tor-dr-replication-start.log-expected.json | 2 +- ...nitor-dr-replication-end.log-expected.json | 2 +- ...7-monitor-fw-rules-start.log-expected.json | 2 +- ...358-monitor-fw-rules-end.log-expected.json | 2 +- ...ault-certificate-is-sha1.log-expected.json | 2 +- ...st-59-clear-safe-history.log-expected.json | 2 +- .../test-88-set-password.log-expected.json | 2 +- .../test-legacysyslog.log-expected.json | 2 +- .../_dev/test/system/test-logfile-config.yml | 2 +- .../_dev/test/system/test-tcp-config.yml | 2 +- .../_dev/test/system/test-tls-config.yml | 2 +- .../_dev/test/system/test-udp-config.yml | 10 +- .../data_stream/audit/fields/beats.yml | 3 - .../data_stream/audit/fields/ecs.yml | 114 ------------------ packages/cyberarkpas/docs/README.md | 66 ---------- packages/cyberarkpas/manifest.yml | 4 +- 19 files changed, 25 insertions(+), 203 deletions(-) delete mode 100644 packages/cyberarkpas/data_stream/audit/fields/ecs.yml diff --git a/packages/cyberarkpas/changelog.yml b/packages/cyberarkpas/changelog.yml index f4784d57d0a..d148fa64198 100644 --- a/packages/cyberarkpas/changelog.yml +++ b/packages/cyberarkpas/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.21.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.20.0" changes: - description: Update manifest format version to v3.0.3. diff --git a/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-288-auto-clear-users-history-start.log-expected.json b/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-288-auto-clear-users-history-start.log-expected.json index 66013d49724..68cf6d1a251 100644 --- a/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-288-auto-clear-users-history-start.log-expected.json +++ b/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-288-auto-clear-users-history-start.log-expected.json @@ -53,7 +53,7 @@ ] }, { - "@timestamp": "2023-03-08T03:00:20.000Z", + "@timestamp": "2024-03-08T03:00:20.000Z", "cyberarkpas": { "audit": { "action": "Auto Clear Users History start", diff --git a/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-289-auto-clear-users-history-end.log-expected.json b/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-289-auto-clear-users-history-end.log-expected.json index fa3a88098d6..97dbbb8feb5 100644 --- a/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-289-auto-clear-users-history-end.log-expected.json +++ b/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-289-auto-clear-users-history-end.log-expected.json @@ -53,7 +53,7 @@ ] }, { - "@timestamp": "2023-03-08T03:00:20.000Z", + "@timestamp": "2024-03-08T03:00:20.000Z", "cyberarkpas": { "audit": { "action": "Auto Clear Users History end", diff --git a/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-310-monitor-dr-replication-start.log-expected.json b/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-310-monitor-dr-replication-start.log-expected.json index 929f7b3bb21..8f39a4f9406 100644 --- a/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-310-monitor-dr-replication-start.log-expected.json +++ b/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-310-monitor-dr-replication-start.log-expected.json @@ -53,7 +53,7 @@ ] }, { - "@timestamp": "2023-03-08T02:48:07.000Z", + "@timestamp": "2024-03-08T02:48:07.000Z", "cyberarkpas": { "audit": { "action": "Monitor DR Replication start", diff --git a/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-311-monitor-dr-replication-end.log-expected.json b/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-311-monitor-dr-replication-end.log-expected.json index 551d75ef5a4..2054f987ba1 100644 --- a/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-311-monitor-dr-replication-end.log-expected.json +++ b/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-311-monitor-dr-replication-end.log-expected.json @@ -53,7 +53,7 @@ ] }, { - "@timestamp": "2023-03-08T02:48:07.000Z", + "@timestamp": "2024-03-08T02:48:07.000Z", "cyberarkpas": { "audit": { "action": "Monitor DR Replication end", diff --git a/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-357-monitor-fw-rules-start.log-expected.json b/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-357-monitor-fw-rules-start.log-expected.json index 6e0f7244c3b..f92147eab77 100644 --- a/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-357-monitor-fw-rules-start.log-expected.json +++ b/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-357-monitor-fw-rules-start.log-expected.json @@ -53,7 +53,7 @@ ] }, { - "@timestamp": "2023-03-08T02:32:56.000Z", + "@timestamp": "2024-03-08T02:32:56.000Z", "cyberarkpas": { "audit": { "action": "Monitor FW rules start", diff --git a/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-358-monitor-fw-rules-end.log-expected.json b/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-358-monitor-fw-rules-end.log-expected.json index b3f6a64238e..c5a4930cdf7 100644 --- a/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-358-monitor-fw-rules-end.log-expected.json +++ b/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-358-monitor-fw-rules-end.log-expected.json @@ -53,7 +53,7 @@ ] }, { - "@timestamp": "2023-03-08T02:32:56.000Z", + "@timestamp": "2024-03-08T02:32:56.000Z", "cyberarkpas": { "audit": { "action": "Monitor FW Rules end", diff --git a/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-479-security-warning-the-signature-hash-algorithm-of-the-vault-certificate-is-sha1.log-expected.json b/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-479-security-warning-the-signature-hash-algorithm-of-the-vault-certificate-is-sha1.log-expected.json index 4784e54a33e..8c01881e357 100644 --- a/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-479-security-warning-the-signature-hash-algorithm-of-the-vault-certificate-is-sha1.log-expected.json +++ b/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-479-security-warning-the-signature-hash-algorithm-of-the-vault-certificate-is-sha1.log-expected.json @@ -56,7 +56,7 @@ ] }, { - "@timestamp": "2023-03-08T07:46:54.000Z", + "@timestamp": "2024-03-08T07:46:54.000Z", "cyberarkpas": { "audit": { "action": "Security warning - The Signature Hash Algorithm of the Vault certificate is SHA1.", diff --git a/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-59-clear-safe-history.log-expected.json b/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-59-clear-safe-history.log-expected.json index 6b6c29a2ff6..c33362d5251 100644 --- a/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-59-clear-safe-history.log-expected.json +++ b/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-59-clear-safe-history.log-expected.json @@ -54,7 +54,7 @@ ] }, { - "@timestamp": "2023-03-08T03:10:31.000Z", + "@timestamp": "2024-03-08T03:10:31.000Z", "cyberarkpas": { "audit": { "action": "Clear Safe History", diff --git a/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-88-set-password.log-expected.json b/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-88-set-password.log-expected.json index 47b88e33a6c..76eab1296f7 100644 --- a/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-88-set-password.log-expected.json +++ b/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-88-set-password.log-expected.json @@ -105,7 +105,7 @@ ] }, { - "@timestamp": "2023-03-08T02:54:46.000Z", + "@timestamp": "2024-03-08T02:54:46.000Z", "cyberarkpas": { "audit": { "action": "Set Password", diff --git a/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-legacysyslog.log-expected.json b/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-legacysyslog.log-expected.json index 10e2241b37b..f66bbb6fde7 100644 --- a/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-legacysyslog.log-expected.json +++ b/packages/cyberarkpas/data_stream/audit/_dev/test/pipeline/test-legacysyslog.log-expected.json @@ -1,7 +1,7 @@ { "expected": [ { - "@timestamp": "2023-03-08T03:41:01.000Z", + "@timestamp": "2024-03-08T03:41:01.000Z", "cyberarkpas": { "audit": { "action": "Retrieve File", diff --git a/packages/cyberarkpas/data_stream/audit/_dev/test/system/test-logfile-config.yml b/packages/cyberarkpas/data_stream/audit/_dev/test/system/test-logfile-config.yml index 8d1df5b6a92..fdb4b0add1d 100644 --- a/packages/cyberarkpas/data_stream/audit/_dev/test/system/test-logfile-config.yml +++ b/packages/cyberarkpas/data_stream/audit/_dev/test/system/test-logfile-config.yml @@ -8,4 +8,4 @@ numeric_keyword_fields: - process.pid - log.syslog.priority assert: - hit_count: 343 \ No newline at end of file + hit_count: 343 diff --git a/packages/cyberarkpas/data_stream/audit/_dev/test/system/test-tcp-config.yml b/packages/cyberarkpas/data_stream/audit/_dev/test/system/test-tcp-config.yml index 69ffe017bc4..7faaf7210d7 100644 --- a/packages/cyberarkpas/data_stream/audit/_dev/test/system/test-tcp-config.yml +++ b/packages/cyberarkpas/data_stream/audit/_dev/test/system/test-tcp-config.yml @@ -6,4 +6,4 @@ data_stream: syslog_host: 0.0.0.0 syslog_port: 9999 assert: - hit_count: 343 \ No newline at end of file + hit_count: 343 diff --git a/packages/cyberarkpas/data_stream/audit/_dev/test/system/test-tls-config.yml b/packages/cyberarkpas/data_stream/audit/_dev/test/system/test-tls-config.yml index 8009c1a76dc..3c5a82dc060 100644 --- a/packages/cyberarkpas/data_stream/audit/_dev/test/system/test-tls-config.yml +++ b/packages/cyberarkpas/data_stream/audit/_dev/test/system/test-tls-config.yml @@ -57,4 +57,4 @@ data_stream: rcZR4kw7O4cWsLR4NHJBosUVoaeoCizBB6xLREqISxIZuHKuEcYsRA== -----END RSA PRIVATE KEY----- assert: - hit_count: 343 \ No newline at end of file + hit_count: 343 diff --git a/packages/cyberarkpas/data_stream/audit/_dev/test/system/test-udp-config.yml b/packages/cyberarkpas/data_stream/audit/_dev/test/system/test-udp-config.yml index fbf6e26e702..c3bcbe0e9ea 100644 --- a/packages/cyberarkpas/data_stream/audit/_dev/test/system/test-udp-config.yml +++ b/packages/cyberarkpas/data_stream/audit/_dev/test/system/test-udp-config.yml @@ -5,8 +5,8 @@ data_stream: vars: syslog_host: 0.0.0.0 syslog_port: 9999 -# Do not assert hit count for this input. Locally, the constraint is -# satisfied, but on CI, apparently the UDP input drops too many (>0) -# messages. -# assert: -# hit_count: 343 \ No newline at end of file + # Do not assert hit count for this input. Locally, the constraint is + # satisfied, but on CI, apparently the UDP input drops too many (>0) + # messages. + # assert: + # hit_count: 343 diff --git a/packages/cyberarkpas/data_stream/audit/fields/beats.yml b/packages/cyberarkpas/data_stream/audit/fields/beats.yml index 9275638f93a..582ff946c0d 100644 --- a/packages/cyberarkpas/data_stream/audit/fields/beats.yml +++ b/packages/cyberarkpas/data_stream/audit/fields/beats.yml @@ -7,9 +7,6 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. - name: log.source.address type: keyword description: Source address from which the log event was read / sent from. diff --git a/packages/cyberarkpas/data_stream/audit/fields/ecs.yml b/packages/cyberarkpas/data_stream/audit/fields/ecs.yml deleted file mode 100644 index 868b5ad3266..00000000000 --- a/packages/cyberarkpas/data_stream/audit/fields/ecs.yml +++ /dev/null @@ -1,114 +0,0 @@ -- external: ecs - name: destination.address -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.domain -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.ip -- external: ecs - name: destination.user.name -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.code -- external: ecs - name: event.duration -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.reason -- external: ecs - name: event.severity -- external: ecs - name: event.timezone -- external: ecs - name: event.type -- external: ecs - name: file.path -- external: ecs - name: host.name -- external: ecs - name: log.syslog.priority -- external: ecs - name: network.application -- external: ecs - name: network.direction -- external: ecs - name: observer.hostname -- external: ecs - name: observer.product -- external: ecs - name: observer.type -- external: ecs - name: observer.vendor -- external: ecs - name: observer.version -- external: ecs - name: process.name -- external: ecs - name: process.pid -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: service.type -- external: ecs - name: source.address -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: source.user.name -- external: ecs - name: tags -- external: ecs - name: user.name -- external: ecs - name: user.target.name diff --git a/packages/cyberarkpas/docs/README.md b/packages/cyberarkpas/docs/README.md index 34eed0b9b66..becacb212e6 100644 --- a/packages/cyberarkpas/docs/README.md +++ b/packages/cyberarkpas/docs/README.md @@ -207,76 +207,10 @@ An example event for `audit` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.user.name | Short name or login of the user. | keyword | -| destination.user.name.text | Multi-field of `destination.user.name`. | match_only_text | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | | event.dataset | Event dataset | constant_keyword | -| event.duration | Duration of the event in nanoseconds. If `event.start` and `event.end` are known this value should be the difference between the end and start time. | long | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Name of the module this data is coming from. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.reason | Reason why this event happened, according to the source. This describes the why of a particular action or outcome captured in the event. Where `event.action` captures the action from the event, `event.reason` describes why that action was taken. For example, a web proxy with an `event.action` which denied the request may also populate `event.reason` with the reason why (e.g. `blocked site`). | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| log.syslog.priority | Syslog numeric priority of the event, if available. According to RFCs 5424 and 3164, the priority is 8 \* facility + severity. This number is therefore expected to contain a value between 0 and 191. | long | -| network.application | When a specific application or service is identified from network connection details (source/dest IPs, ports, certificates, or wire format), this field captures the application's or service's name. For example, the original event identifies the network connection being from a specific web service in a `https` network connection, like `facebook` or `twitter`. The field value must be normalized to lowercase for querying. | keyword | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| observer.hostname | Hostname of the observer. | keyword | -| observer.product | The product name of the observer. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| observer.version | Observer version. | keyword | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.pid | Process id. | long | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.user.name | Short name or login of the user. | keyword | -| source.user.name.text | Multi-field of `source.user.name`. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user.target.name | Short name or login of the user. | keyword | -| user.target.name.text | Multi-field of `user.target.name`. | match_only_text | diff --git a/packages/cyberarkpas/manifest.yml b/packages/cyberarkpas/manifest.yml index 248acd14c07..4fa3cbeeb2d 100644 --- a/packages/cyberarkpas/manifest.yml +++ b/packages/cyberarkpas/manifest.yml @@ -1,13 +1,13 @@ name: cyberarkpas title: CyberArk Privileged Access Security -version: "2.20.0" +version: "2.21.0" description: Collect logs from CyberArk Privileged Access Security with Elastic Agent. type: integration format_version: "3.0.3" categories: ["security", "iam"] conditions: kibana: - version: ^8.7.1 + version: "^8.13.0" screenshots: - src: /img/filebeat-cyberarkpas-overview.png title: filebeat cyberarkpas overview From c27b3a3892ff7188fec1bf3f5847cd6f8a5c999e Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:24:18 +0930 Subject: [PATCH 032/121] [cyberark_pta] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^7.17.0 || ^8.0.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/cyberark_pta --- packages/cyberark_pta/changelog.yml | 5 +++ .../data_stream/events/fields/ecs.yml | 38 ------------------- packages/cyberark_pta/docs/README.md | 21 ---------- packages/cyberark_pta/manifest.yml | 4 +- 4 files changed, 7 insertions(+), 61 deletions(-) delete mode 100644 packages/cyberark_pta/data_stream/events/fields/ecs.yml diff --git a/packages/cyberark_pta/changelog.yml b/packages/cyberark_pta/changelog.yml index 8e4720b2b39..518877773d5 100644 --- a/packages/cyberark_pta/changelog.yml +++ b/packages/cyberark_pta/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.10.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.9.0" changes: - description: Update manifest format version to v3.0.3. diff --git a/packages/cyberark_pta/data_stream/events/fields/ecs.yml b/packages/cyberark_pta/data_stream/events/fields/ecs.yml deleted file mode 100644 index a3391d494bb..00000000000 --- a/packages/cyberark_pta/data_stream/events/fields/ecs.yml +++ /dev/null @@ -1,38 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: observer.vendor -- external: ecs - name: observer.product -- external: ecs - name: observer.version -- external: ecs - name: event.reason -- external: ecs - name: event.severity -- external: ecs - name: source.user.name -- external: ecs - name: source.domain -- external: ecs - name: source.ip -- external: ecs - name: destination.user.name -- external: ecs - name: destination.domain -- external: ecs - name: destination.ip -- external: ecs - name: event.id -- external: ecs - name: event.created -- external: ecs - name: event.reference -- external: ecs - name: event.url -- external: ecs - name: event.action -- external: ecs - name: message -- external: ecs - name: tags diff --git a/packages/cyberark_pta/docs/README.md b/packages/cyberark_pta/docs/README.md index 0c6e2eeb634..42dc6f88fd9 100644 --- a/packages/cyberark_pta/docs/README.md +++ b/packages/cyberark_pta/docs/README.md @@ -280,30 +280,9 @@ An example event for pta looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | | destination.service.name | | keyword | -| destination.user.name | Short name or login of the user. | keyword | -| destination.user.name.text | Multi-field of `destination.user.name`. | match_only_text | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | | event.module | Event module | constant_keyword | -| event.reason | Reason why this event happened, according to the source. This describes the why of a particular action or outcome captured in the event. Where `event.action` captures the action from the event, `event.reason` describes why that action was taken. For example, a web proxy with an `event.action` which denied the request may also populate `event.reason` with the reason why (e.g. `blocked site`). | keyword | -| event.reference | Reference URL linking to additional information about this event. This URL links to a static definition of this event. Alert events, indicated by `event.kind:alert`, are a common use case for this field. | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.url | URL linking to an external system to continue investigation of this event. This URL links to another system where in-depth investigation of the specific occurrence of this event can take place. Alert events, indicated by `event.kind:alert`, are a common use case for this field. | keyword | | input.type | Input type | keyword | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| observer.product | The product name of the observer. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| observer.version | Observer version. | keyword | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | | source.service.name | | keyword | -| source.user.name | Short name or login of the user. | keyword | -| source.user.name.text | Multi-field of `source.user.name`. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | diff --git a/packages/cyberark_pta/manifest.yml b/packages/cyberark_pta/manifest.yml index de0895d9166..4174cdcb71f 100644 --- a/packages/cyberark_pta/manifest.yml +++ b/packages/cyberark_pta/manifest.yml @@ -1,13 +1,13 @@ name: cyberark_pta title: Cyberark Privileged Threat Analytics -version: "1.9.0" +version: "1.10.0" description: Collect security logs from Cyberark PTA integration. type: integration format_version: "3.0.3" categories: ["security", "iam"] conditions: kibana: - version: ^7.17.0 || ^8.0.0 + version: "^8.13.0" screenshots: - src: /img/cyberarkpta-overview.png title: cyberark pta overview From 616c9fdf2dff20ccffe3e4f6772afefe1d363c03 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:24:23 +0930 Subject: [PATCH 033/121] [cybereason] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/cybereason --- packages/cybereason/_dev/build/build.yml | 1 - packages/cybereason/changelog.yml | 5 +++++ .../cybereason/data_stream/logon_session/fields/beats.yml | 3 --- .../data_stream/malop_connection/fields/beats.yml | 3 --- .../cybereason/data_stream/malop_process/fields/beats.yml | 3 --- packages/cybereason/data_stream/malware/fields/beats.yml | 3 --- packages/cybereason/data_stream/poll_malop/fields/beats.yml | 3 --- .../data_stream/suspicions_process/fields/beats.yml | 3 --- packages/cybereason/docs/README.md | 6 ------ packages/cybereason/manifest.yml | 4 ++-- 10 files changed, 7 insertions(+), 27 deletions(-) diff --git a/packages/cybereason/_dev/build/build.yml b/packages/cybereason/_dev/build/build.yml index 1f4fa988f6e..e2b012548e0 100644 --- a/packages/cybereason/_dev/build/build.yml +++ b/packages/cybereason/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: git@v8.11.0 - import_mappings: true diff --git a/packages/cybereason/changelog.yml b/packages/cybereason/changelog.yml index ee44ece1fd3..dee7eaca8df 100644 --- a/packages/cybereason/changelog.yml +++ b/packages/cybereason/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.2.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "0.1.0" changes: - description: Initial release. diff --git a/packages/cybereason/data_stream/logon_session/fields/beats.yml b/packages/cybereason/data_stream/logon_session/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/cybereason/data_stream/logon_session/fields/beats.yml +++ b/packages/cybereason/data_stream/logon_session/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/cybereason/data_stream/malop_connection/fields/beats.yml b/packages/cybereason/data_stream/malop_connection/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/cybereason/data_stream/malop_connection/fields/beats.yml +++ b/packages/cybereason/data_stream/malop_connection/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/cybereason/data_stream/malop_process/fields/beats.yml b/packages/cybereason/data_stream/malop_process/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/cybereason/data_stream/malop_process/fields/beats.yml +++ b/packages/cybereason/data_stream/malop_process/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/cybereason/data_stream/malware/fields/beats.yml b/packages/cybereason/data_stream/malware/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/cybereason/data_stream/malware/fields/beats.yml +++ b/packages/cybereason/data_stream/malware/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/cybereason/data_stream/poll_malop/fields/beats.yml b/packages/cybereason/data_stream/poll_malop/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/cybereason/data_stream/poll_malop/fields/beats.yml +++ b/packages/cybereason/data_stream/poll_malop/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/cybereason/data_stream/suspicions_process/fields/beats.yml b/packages/cybereason/data_stream/suspicions_process/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/cybereason/data_stream/suspicions_process/fields/beats.yml +++ b/packages/cybereason/data_stream/suspicions_process/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/cybereason/docs/README.md b/packages/cybereason/docs/README.md index 2b2aa6b7093..c4beb659713 100644 --- a/packages/cybereason/docs/README.md +++ b/packages/cybereason/docs/README.md @@ -336,7 +336,6 @@ An example event for `logon_session` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Malop Connection @@ -708,7 +707,6 @@ An example event for `malop_connection` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Malop Process @@ -2193,7 +2191,6 @@ An example event for `malop_process` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Malware @@ -2319,7 +2316,6 @@ An example event for `malware` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Poll Malop @@ -2565,7 +2561,6 @@ An example event for `poll_malop` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Suspicions Process @@ -3019,4 +3014,3 @@ An example event for `suspicions_process` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | diff --git a/packages/cybereason/manifest.yml b/packages/cybereason/manifest.yml index 2122606f268..e3bbf2bbe94 100644 --- a/packages/cybereason/manifest.yml +++ b/packages/cybereason/manifest.yml @@ -1,14 +1,14 @@ format_version: 3.0.3 name: cybereason title: Cybereason -version: 0.1.0 +version: "0.2.0" description: Collect logs from Cybereason with Elastic Agent. type: integration categories: - security conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" elastic: subscription: basic screenshots: From 406a7704fb2d6c70ff6e4d2429e6144eb656bc68 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:24:24 +0930 Subject: [PATCH 034/121] [cylance] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^7.14.1 || ^8.0.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/cylance --- packages/cylance/changelog.yml | 5 + .../protect/fields/base-fields.yml | 14 -- .../data_stream/protect/fields/ecs.yml | 216 ------------------ packages/cylance/docs/README.md | 122 ---------- packages/cylance/manifest.yml | 4 +- 5 files changed, 7 insertions(+), 354 deletions(-) diff --git a/packages/cylance/changelog.yml b/packages/cylance/changelog.yml index 61347300f85..7d7fb9be13e 100644 --- a/packages/cylance/changelog.yml +++ b/packages/cylance/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.20.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "0.19.3" changes: - description: Fix `kibana.version` syntax in manifest. diff --git a/packages/cylance/data_stream/protect/fields/base-fields.yml b/packages/cylance/data_stream/protect/fields/base-fields.yml index 0e6d6970311..665aacc3d98 100644 --- a/packages/cylance/data_stream/protect/fields/base-fields.yml +++ b/packages/cylance/data_stream/protect/fields/base-fields.yml @@ -15,18 +15,9 @@ type: constant_keyword description: Event dataset value: cylance.protect -- name: container.id - description: Unique container id. - ignore_above: 1024 - type: keyword - name: input.type description: Type of Filebeat input. type: keyword -- name: log.file.path - description: Full path to the log file this event came from. - example: /var/log/fun-times.log - ignore_above: 1024 - type: keyword - name: log.source.address description: Source address from which the log event was read / sent from. type: keyword @@ -36,8 +27,3 @@ - name: log.offset description: Offset of the entry in the log file. type: long -- name: tags - description: List of keywords used to tag each event. - example: '["production", "env2"]' - ignore_above: 1024 - type: keyword diff --git a/packages/cylance/data_stream/protect/fields/ecs.yml b/packages/cylance/data_stream/protect/fields/ecs.yml index 5d22b129aef..6ce5c3d0b78 100644 --- a/packages/cylance/data_stream/protect/fields/ecs.yml +++ b/packages/cylance/data_stream/protect/fields/ecs.yml @@ -1,87 +1,5 @@ - external: ecs name: '@timestamp' -- external: ecs - name: client.domain -- external: ecs - name: client.registered_domain -- external: ecs - name: client.subdomain -- external: ecs - name: client.top_level_domain -- external: ecs - name: destination.address -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.bytes -- external: ecs - name: destination.domain -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.ip -- external: ecs - name: destination.mac -- external: ecs - name: destination.nat.ip -- external: ecs - name: destination.nat.port -- external: ecs - name: destination.port -- external: ecs - name: destination.registered_domain -- external: ecs - name: destination.subdomain -- external: ecs - name: destination.top_level_domain -- external: ecs - name: dns.answers.name -- external: ecs - name: dns.answers.type -- external: ecs - name: dns.question.registered_domain -- external: ecs - name: dns.question.subdomain -- external: ecs - name: dns.question.top_level_domain -- external: ecs - name: dns.question.type -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.code -- external: ecs - name: event.ingested -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.timezone -- external: ecs - name: file.attributes -- external: ecs - name: file.directory -- external: ecs - name: file.extension -- external: ecs - name: file.name -- external: ecs - name: file.path -- external: ecs - name: file.size -- external: ecs - name: file.type - external: ecs name: geo.city_name - external: ecs @@ -90,137 +8,3 @@ name: geo.name - external: ecs name: geo.region_name -- external: ecs - name: group.id -- external: ecs - name: group.name -- external: ecs - name: host.hostname -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: http.request.method -- external: ecs - name: http.request.referrer -- external: ecs - name: log.level -- external: ecs - name: log.syslog.facility.code -- external: ecs - name: log.syslog.priority -- external: ecs - name: log.syslog.severity.code -- external: ecs - name: message -- external: ecs - name: network.application -- external: ecs - name: network.bytes -- external: ecs - name: network.direction -- external: ecs - name: network.forwarded_ip -- external: ecs - name: network.packets -- external: ecs - name: network.protocol -- external: ecs - name: observer.egress.interface.name -- external: ecs - name: observer.ingress.interface.name -- external: ecs - name: observer.product -- external: ecs - name: observer.type -- external: ecs - name: observer.vendor -- external: ecs - name: observer.version -- external: ecs - name: process.name -- external: ecs - name: process.parent.name -- external: ecs - name: process.parent.title -- external: ecs - name: process.pid -- external: ecs - name: process.parent.pid -- external: ecs - name: process.title -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: rule.name -- external: ecs - name: server.domain -- external: ecs - name: server.registered_domain -- external: ecs - name: server.subdomain -- external: ecs - name: server.top_level_domain -- external: ecs - name: service.name -- external: ecs - name: source.address -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.bytes -- external: ecs - name: source.domain -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.ip -- external: ecs - name: source.mac -- external: ecs - name: source.nat.ip -- external: ecs - name: source.nat.port -- external: ecs - name: source.port -- external: ecs - name: source.registered_domain -- external: ecs - name: source.subdomain -- external: ecs - name: source.top_level_domain -- external: ecs - name: url.domain -- external: ecs - name: url.original -- external: ecs - name: url.path -- external: ecs - name: url.query -- external: ecs - name: url.registered_domain -- external: ecs - name: url.top_level_domain -- external: ecs - name: user.domain -- external: ecs - name: user.full_name -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user_agent.original diff --git a/packages/cylance/docs/README.md b/packages/cylance/docs/README.md index 494b8fd9007..08325a6516b 100644 --- a/packages/cylance/docs/README.md +++ b/packages/cylance/docs/README.md @@ -13,104 +13,21 @@ The `protect` dataset collects CylanceProtect logs. | Field | Description | Type | |---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | -| client.domain | The domain name of the client system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| client.registered_domain | The highest registered client domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| client.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| client.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| container.id | Unique container id. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.bytes | Bytes sent from the destination to the source. | long | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.mac | MAC address of the destination. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| destination.nat.ip | Translated ip of destination based NAT sessions (e.g. internet to private DMZ) Typically used with load balancers, firewalls, or routers. | ip | -| destination.nat.port | Port the source session is translated to by NAT Device. Typically used with load balancers, firewalls, or routers. | long | -| destination.port | Port of the destination. | long | -| destination.registered_domain | The highest registered destination domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| destination.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| destination.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| dns.answers.name | The domain name to which this resource record pertains. If a chain of CNAME is being resolved, each answer's `name` should be the one that corresponds with the answer's `data`. It should not simply be the original `question.name` repeated. | keyword | -| dns.answers.type | The type of data contained in this resource record. | keyword | | dns.question.domain | Server domain. | keyword | -| dns.question.registered_domain | The highest registered domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| dns.question.subdomain | The subdomain is all of the labels under the registered_domain. If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| dns.question.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| dns.question.type | The type of record being queried. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| file.attributes | Array of file attributes. Attributes names will vary by platform. Here's a non-exhaustive list of values that are expected in this field: archive, compressed, directory, encrypted, execute, hidden, read, readonly, system, write. | keyword | -| file.directory | Directory where the file is located. It should include the drive letter, when appropriate. | keyword | -| file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| file.type | File type (file, dir, or symlink). | keyword | | geo.city_name | City name. | keyword | | geo.country_name | Country name. | keyword | | geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | | geo.region_name | Region name. | keyword | -| group.id | Unique identifier for the group on the system/platform. | keyword | -| group.name | Name of the group. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.request.referrer | Referrer for this HTTP request. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Full path to the log file this event came from. | keyword | | log.flags | Flags for the log file. | keyword | -| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | | log.offset | Offset of the entry in the log file. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| log.syslog.facility.code | The Syslog numeric facility of the log event, if available. According to RFCs 5424 and 3164, this value should be an integer between 0 and 23. | long | -| log.syslog.priority | Syslog numeric priority of the event, if available. According to RFCs 5424 and 3164, the priority is 8 \* facility + severity. This number is therefore expected to contain a value between 0 and 191. | long | -| log.syslog.severity.code | The Syslog numeric severity of the log event, if available. If the event source publishing via Syslog provides a different numeric severity value (e.g. firewall, IDS), your source's numeric severity should go to `event.severity`. If the event source does not specify a distinct severity, you can optionally copy the Syslog severity to `event.severity`. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.application | When a specific application or service is identified from network connection details (source/dest IPs, ports, certificates, or wire format), this field captures the application's or service's name. For example, the original event identifies the network connection being from a specific web service in a `https` network connection, like `facebook` or `twitter`. The field value must be normalized to lowercase for querying. | keyword | -| network.bytes | Total bytes transferred in both directions. If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. | long | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.forwarded_ip | Host IP address when the source IP address is the proxy. | ip | | network.interface.name | | keyword | -| network.packets | Total packets transferred in both directions. If `source.packets` and `destination.packets` are known, `network.packets` is their sum. | long | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| observer.egress.interface.name | Interface name as reported by the system. | keyword | -| observer.ingress.interface.name | Interface name as reported by the system. | keyword | -| observer.product | The product name of the observer. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| observer.version | Observer version. | keyword | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.parent.name | Process name. Sometimes called program name or similar. | keyword | -| process.parent.name.text | Multi-field of `process.parent.name`. | match_only_text | -| process.parent.pid | Process id. | long | -| process.parent.title | Process title. The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. | keyword | -| process.parent.title.text | Multi-field of `process.parent.title`. | match_only_text | -| process.pid | Process id. | long | -| process.title | Process title. The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. | keyword | -| process.title.text | Multi-field of `process.title`. | match_only_text | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | | rsa.counters.dclass_c1 | This is a generic counter key that should be used with the label dclass.c1.str only | long | | rsa.counters.dclass_c1_str | This is a generic counter string key that should be used with the label dclass.c1 only | keyword | | rsa.counters.dclass_c2 | This is a generic counter key that should be used with the label dclass.c2.str only | long | @@ -783,42 +700,3 @@ The `protect` dataset collects CylanceProtect logs. | rsa.wireless.wlan_channel | This is used to capture the channel names | long | | rsa.wireless.wlan_name | This key captures either WLAN number/name | keyword | | rsa.wireless.wlan_ssid | This key is used to capture the ssid of a Wireless Session | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| server.domain | The domain name of the server system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| server.registered_domain | The highest registered server domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| server.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| server.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.mac | MAC address of the source. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| source.nat.ip | Translated ip of source based NAT sessions (e.g. internal client to internet) Typically connections traversing load balancers, firewalls, or routers. | ip | -| source.nat.port | Translated port of source based NAT sessions. (e.g. internal client to internet) Typically used with load balancers, firewalls, or routers. | long | -| source.port | Port of the source. | long | -| source.registered_domain | The highest registered source domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| source.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| source.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | diff --git a/packages/cylance/manifest.yml b/packages/cylance/manifest.yml index bf31368efe1..fd17850c1ef 100644 --- a/packages/cylance/manifest.yml +++ b/packages/cylance/manifest.yml @@ -1,13 +1,13 @@ format_version: 2.7.0 name: cylance title: CylanceProtect Logs -version: "0.19.3" +version: "0.20.0" description: Collect logs from CylanceProtect devices with Elastic Agent. categories: ["security", "edr_xdr"] type: integration conditions: kibana: - version: "^7.14.1 || ^8.0.0" + version: "^8.13.0" policy_templates: - name: protect title: CylanceProtect From 0cf0d664ae3a8251c6b83781c6ff725e384aa705 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:24:27 +0930 Subject: [PATCH 035/121] [darktrace] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/darktrace --- packages/darktrace/changelog.yml | 5 + .../ai_analyst_alert/fields/agent.yml | 128 -------------- .../ai_analyst_alert/fields/ecs.yml | 64 ------- .../model_breach_alert/fields/agent.yml | 128 -------------- .../model_breach_alert/fields/ecs.yml | 74 -------- .../system_status_alert/fields/agent.yml | 138 --------------- .../system_status_alert/fields/ecs.yml | 48 ------ packages/darktrace/docs/README.md | 162 ------------------ packages/darktrace/manifest.yml | 4 +- 9 files changed, 7 insertions(+), 744 deletions(-) delete mode 100644 packages/darktrace/data_stream/ai_analyst_alert/fields/ecs.yml delete mode 100644 packages/darktrace/data_stream/model_breach_alert/fields/ecs.yml delete mode 100644 packages/darktrace/data_stream/system_status_alert/fields/ecs.yml diff --git a/packages/darktrace/changelog.yml b/packages/darktrace/changelog.yml index c8bc1e4924d..8d28b1412b4 100644 --- a/packages/darktrace/changelog.yml +++ b/packages/darktrace/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.18.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.17.0" changes: - description: Make `host.mac` field conform to ECS field definition. diff --git a/packages/darktrace/data_stream/ai_analyst_alert/fields/agent.yml b/packages/darktrace/data_stream/ai_analyst_alert/fields/agent.yml index 47d5be58da9..d3d659d48f2 100644 --- a/packages/darktrace/data_stream/ai_analyst_alert/fields/agent.yml +++ b/packages/darktrace/data_stream/ai_analyst_alert/fields/agent.yml @@ -5,143 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: >- diff --git a/packages/darktrace/data_stream/ai_analyst_alert/fields/ecs.yml b/packages/darktrace/data_stream/ai_analyst_alert/fields/ecs.yml deleted file mode 100644 index 4fcab038289..00000000000 --- a/packages/darktrace/data_stream/ai_analyst_alert/fields/ecs.yml +++ /dev/null @@ -1,64 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.end -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.reason -- external: ecs - name: event.risk_score -- external: ecs - name: event.risk_score_norm -- external: ecs - name: event.start -- external: ecs - name: event.type -- external: ecs - name: event.url -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.name -- external: ecs - name: log.syslog.appname -- external: ecs - name: log.syslog.facility.code -- external: ecs - name: log.syslog.facility.name -- external: ecs - name: log.syslog.hostname -- external: ecs - name: log.syslog.priority -- external: ecs - name: log.syslog.severity.code -- external: ecs - name: log.syslog.severity.name -- external: ecs - name: log.syslog.version -- external: ecs - name: message -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: rule.name -- external: ecs - name: tags -- external: ecs - name: threat.enrichments.matched.id -- external: ecs - name: threat.group.id diff --git a/packages/darktrace/data_stream/model_breach_alert/fields/agent.yml b/packages/darktrace/data_stream/model_breach_alert/fields/agent.yml index 2ad539b9eb2..89c81d2ed1c 100644 --- a/packages/darktrace/data_stream/model_breach_alert/fields/agent.yml +++ b/packages/darktrace/data_stream/model_breach_alert/fields/agent.yml @@ -5,143 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - name: containerized type: boolean description: >- diff --git a/packages/darktrace/data_stream/model_breach_alert/fields/ecs.yml b/packages/darktrace/data_stream/model_breach_alert/fields/ecs.yml deleted file mode 100644 index 2b34237b623..00000000000 --- a/packages/darktrace/data_stream/model_breach_alert/fields/ecs.yml +++ /dev/null @@ -1,74 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.risk_score -- external: ecs - name: event.risk_score_norm -- external: ecs - name: event.severity -- external: ecs - name: event.start -- external: ecs - name: event.type -- external: ecs - name: event.url -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.type -- external: ecs - name: log.syslog.appname -- external: ecs - name: log.syslog.facility.code -- external: ecs - name: log.syslog.facility.name -- external: ecs - name: log.syslog.hostname -- external: ecs - name: log.syslog.priority -- external: ecs - name: log.syslog.severity.code -- external: ecs - name: log.syslog.severity.name -- external: ecs - name: log.syslog.version -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: rule.author -- external: ecs - name: rule.category -- external: ecs - name: rule.description -- external: ecs - name: rule.name -- external: ecs - name: rule.ruleset -- external: ecs - name: rule.uuid -- external: ecs - name: rule.version -- external: ecs - name: tags -- external: ecs - name: threat.technique.id -- external: ecs - name: threat.technique.name diff --git a/packages/darktrace/data_stream/system_status_alert/fields/agent.yml b/packages/darktrace/data_stream/system_status_alert/fields/agent.yml index feb71b5a75f..d3d659d48f2 100644 --- a/packages/darktrace/data_stream/system_status_alert/fields/agent.yml +++ b/packages/darktrace/data_stream/system_status_alert/fields/agent.yml @@ -5,153 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: >- diff --git a/packages/darktrace/data_stream/system_status_alert/fields/ecs.yml b/packages/darktrace/data_stream/system_status_alert/fields/ecs.yml deleted file mode 100644 index f2acb655a91..00000000000 --- a/packages/darktrace/data_stream/system_status_alert/fields/ecs.yml +++ /dev/null @@ -1,48 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.reason -- external: ecs - name: event.risk_score -- external: ecs - name: event.risk_score_norm -- external: ecs - name: event.type -- external: ecs - name: event.url -- external: ecs - name: host.hostname -- external: ecs - name: host.ip -- external: ecs - name: log.syslog.appname -- external: ecs - name: log.syslog.facility.code -- external: ecs - name: log.syslog.facility.name -- external: ecs - name: log.syslog.hostname -- external: ecs - name: log.syslog.priority -- external: ecs - name: log.syslog.severity.code -- external: ecs - name: log.syslog.severity.name -- external: ecs - name: log.syslog.version -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: tags diff --git a/packages/darktrace/docs/README.md b/packages/darktrace/docs/README.md index d53a8e5530f..2ea9d7a4ba7 100644 --- a/packages/darktrace/docs/README.md +++ b/packages/darktrace/docs/README.md @@ -357,19 +357,7 @@ An example event for `ai_analyst_alert` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | darktrace.ai_analyst_alert.activity_id | An identifier for the specific activity detected by AI Analyst. If groupByActivity=true , this field should be used to group events together into an incident. | keyword | | darktrace.ai_analyst_alert.aia_score | The score of the event as classified by AI Analyst - out of 100. | double | | darktrace.ai_analyst_alert.attack_phases | Of the six attack phases, which phases are applicable to the activity. | long | @@ -418,56 +406,14 @@ An example event for `ai_analyst_alert` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.reason | Reason why this event happened, according to the source. This describes the why of a particular action or outcome captured in the event. Where `event.action` captures the action from the event, `event.reason` describes why that action was taken. For example, a web proxy with an `event.action` which denied the request may also populate `event.reason` with the reason why (e.g. `blocked site`). | keyword | -| event.risk_score | Risk score or priority of the event (e.g. security solutions). Use your system's original value here. | float | -| event.risk_score_norm | Normalized risk score or priority of the event, on a scale of 0 to 100. This is mainly useful if you use more than one system that assigns risk scores, and you want to see a normalized value across all systems. | float | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| event.url | URL linking to an external system to continue investigation of this event. This URL links to another system where in-depth investigation of the specific occurrence of this event can take place. Alert events, indicated by `event.kind:alert`, are a common use case for this field. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| log.syslog.appname | The device or application that originated the Syslog message, if available. | keyword | -| log.syslog.facility.code | The Syslog numeric facility of the log event, if available. According to RFCs 5424 and 3164, this value should be an integer between 0 and 23. | long | -| log.syslog.facility.name | The Syslog text-based facility of the log event, if available. | keyword | -| log.syslog.hostname | The hostname, FQDN, or IP of the machine that originally sent the Syslog message. This is sourced from the hostname field of the syslog header. Depending on the environment, this value may be different from the host that handled the event, especially if the host handling the events is acting as a collector. | keyword | -| log.syslog.priority | Syslog numeric priority of the event, if available. According to RFCs 5424 and 3164, the priority is 8 \* facility + severity. This number is therefore expected to contain a value between 0 and 191. | long | -| log.syslog.severity.code | The Syslog numeric severity of the log event, if available. If the event source publishing via Syslog provides a different numeric severity value (e.g. firewall, IDS), your source's numeric severity should go to `event.severity`. If the event source does not specify a distinct severity, you can optionally copy the Syslog severity to `event.severity`. | long | -| log.syslog.severity.name | The Syslog numeric severity of the log event, if available. If the event source publishing via Syslog provides a different severity value (e.g. firewall, IDS), your source's text severity should go to `log.level`. If the event source does not specify a distinct severity, you can optionally copy the Syslog severity to `log.level`. | keyword | -| log.syslog.version | The version of the Syslog protocol specification. Only applicable for RFC 5424 messages. | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| rule.name | The name of the rule or signature generating the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| threat.enrichments.matched.id | Identifies the _id of the indicator document enriching the event. | keyword | -| threat.group.id | The id of the group for a set of related intrusion activity that are tracked by a common name in the security community. While not required, you can use a MITRE ATT&CK® group id. | keyword | ### model_breach_alert @@ -1072,19 +1018,7 @@ An example event for `model_breach_alert` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | darktrace.model_breach_alert.aianalyst_data.related | | long | | darktrace.model_breach_alert.aianalyst_data.summariser | | keyword | | darktrace.model_breach_alert.aianalyst_data.uuid | | keyword | @@ -1214,62 +1148,14 @@ An example event for `model_breach_alert` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.risk_score | Risk score or priority of the event (e.g. security solutions). Use your system's original value here. | float | -| event.risk_score_norm | Normalized risk score or priority of the event, on a scale of 0 to 100. This is mainly useful if you use more than one system that assigns risk scores, and you want to see a normalized value across all systems. | float | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| event.url | URL linking to an external system to continue investigation of this event. This URL links to another system where in-depth investigation of the specific occurrence of this event can take place. Alert events, indicated by `event.kind:alert`, are a common use case for this field. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| log.syslog.appname | The device or application that originated the Syslog message, if available. | keyword | -| log.syslog.facility.code | The Syslog numeric facility of the log event, if available. According to RFCs 5424 and 3164, this value should be an integer between 0 and 23. | long | -| log.syslog.facility.name | The Syslog text-based facility of the log event, if available. | keyword | -| log.syslog.hostname | The hostname, FQDN, or IP of the machine that originally sent the Syslog message. This is sourced from the hostname field of the syslog header. Depending on the environment, this value may be different from the host that handled the event, especially if the host handling the events is acting as a collector. | keyword | -| log.syslog.priority | Syslog numeric priority of the event, if available. According to RFCs 5424 and 3164, the priority is 8 \* facility + severity. This number is therefore expected to contain a value between 0 and 191. | long | -| log.syslog.severity.code | The Syslog numeric severity of the log event, if available. If the event source publishing via Syslog provides a different numeric severity value (e.g. firewall, IDS), your source's numeric severity should go to `event.severity`. If the event source does not specify a distinct severity, you can optionally copy the Syslog severity to `event.severity`. | long | -| log.syslog.severity.name | The Syslog numeric severity of the log event, if available. If the event source publishing via Syslog provides a different severity value (e.g. firewall, IDS), your source's text severity should go to `log.level`. If the event source does not specify a distinct severity, you can optionally copy the Syslog severity to `log.level`. | keyword | -| log.syslog.version | The version of the Syslog protocol specification. Only applicable for RFC 5424 messages. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.author | Name, organization, or pseudonym of the author or authors who created the rule used to generate this event. | keyword | -| rule.category | A categorization value keyword used by the entity using the rule for detection of this event. | keyword | -| rule.description | The description of the rule generating the event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| rule.ruleset | Name of the ruleset, policy, group, or parent category in which the rule used to generate this event is a member. | keyword | -| rule.uuid | A rule ID that is unique within the scope of a set or group of agents, observers, or other entities using the rule for detection of this event. | keyword | -| rule.version | The version / revision of the rule being used for analysis. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| threat.technique.id | The id of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.name | The name of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.name.text | Multi-field of `threat.technique.name`. | match_only_text | ### system_status_alert @@ -1383,19 +1269,7 @@ An example event for `system_status_alert` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | darktrace.system_status_alert.acknowledge_timeout | When acknowledgement of the alert expires. As alerts are sent externally on creation before acknowledgement is possible, this will be null in almost all cases. | keyword | | darktrace.system_status_alert.alert_name | A human readable name of the alert type. | keyword | | darktrace.system_status_alert.child_id | For probes (physical or virtual), the unique ID associated with the probe. | long | @@ -1423,47 +1297,11 @@ An example event for `system_status_alert` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.reason | Reason why this event happened, according to the source. This describes the why of a particular action or outcome captured in the event. Where `event.action` captures the action from the event, `event.reason` describes why that action was taken. For example, a web proxy with an `event.action` which denied the request may also populate `event.reason` with the reason why (e.g. `blocked site`). | keyword | -| event.risk_score | Risk score or priority of the event (e.g. security solutions). Use your system's original value here. | float | -| event.risk_score_norm | Normalized risk score or priority of the event, on a scale of 0 to 100. This is mainly useful if you use more than one system that assigns risk scores, and you want to see a normalized value across all systems. | float | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| event.url | URL linking to an external system to continue investigation of this event. This URL links to another system where in-depth investigation of the specific occurrence of this event can take place. Alert events, indicated by `event.kind:alert`, are a common use case for this field. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| log.syslog.appname | The device or application that originated the Syslog message, if available. | keyword | -| log.syslog.facility.code | The Syslog numeric facility of the log event, if available. According to RFCs 5424 and 3164, this value should be an integer between 0 and 23. | long | -| log.syslog.facility.name | The Syslog text-based facility of the log event, if available. | keyword | -| log.syslog.hostname | The hostname, FQDN, or IP of the machine that originally sent the Syslog message. This is sourced from the hostname field of the syslog header. Depending on the environment, this value may be different from the host that handled the event, especially if the host handling the events is acting as a collector. | keyword | -| log.syslog.priority | Syslog numeric priority of the event, if available. According to RFCs 5424 and 3164, the priority is 8 \* facility + severity. This number is therefore expected to contain a value between 0 and 191. | long | -| log.syslog.severity.code | The Syslog numeric severity of the log event, if available. If the event source publishing via Syslog provides a different numeric severity value (e.g. firewall, IDS), your source's numeric severity should go to `event.severity`. If the event source does not specify a distinct severity, you can optionally copy the Syslog severity to `event.severity`. | long | -| log.syslog.severity.name | The Syslog numeric severity of the log event, if available. If the event source publishing via Syslog provides a different severity value (e.g. firewall, IDS), your source's text severity should go to `log.level`. If the event source does not specify a distinct severity, you can optionally copy the Syslog severity to `log.level`. | keyword | -| log.syslog.version | The version of the Syslog protocol specification. Only applicable for RFC 5424 messages. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| tags | List of keywords used to tag each event. | keyword | diff --git a/packages/darktrace/manifest.yml b/packages/darktrace/manifest.yml index 750fa88fd76..0d916f6bbd0 100644 --- a/packages/darktrace/manifest.yml +++ b/packages/darktrace/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: darktrace title: Darktrace -version: "1.17.0" +version: "1.18.0" description: Collect logs from Darktrace with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - network_security conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" screenshots: - src: /img/darktrace-screenshot.png title: Darktrace Model Breach Alert Dashboard Screenshot From 4e5db15c285f4fcc4ca5eed7dbc7b86cb4a55949 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:24:28 +0930 Subject: [PATCH 036/121] [entityanalytics_ad] - removed ecs import_mappings Removed import_mappings. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/entityanalytics_ad --- packages/entityanalytics_ad/_dev/build/build.yml | 1 - packages/entityanalytics_ad/changelog.yml | 5 +++++ .../user/_dev/test/pipeline/test-user.json | 11 ++++++++++- .../data_stream/user/fields/beats.yml | 3 --- .../data_stream/user/fields/fields.yml | 1 - packages/entityanalytics_ad/docs/README.md | 1 - packages/entityanalytics_ad/manifest.yml | 2 +- 7 files changed, 16 insertions(+), 8 deletions(-) diff --git a/packages/entityanalytics_ad/_dev/build/build.yml b/packages/entityanalytics_ad/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/entityanalytics_ad/_dev/build/build.yml +++ b/packages/entityanalytics_ad/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/entityanalytics_ad/changelog.yml b/packages/entityanalytics_ad/changelog.yml index e67c14252eb..cd042752c93 100644 --- a/packages/entityanalytics_ad/changelog.yml +++ b/packages/entityanalytics_ad/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.1.0" + changes: + - description: Removed import_mappings. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "0.0.1" changes: - description: Initial Release. diff --git a/packages/entityanalytics_ad/data_stream/user/_dev/test/pipeline/test-user.json b/packages/entityanalytics_ad/data_stream/user/_dev/test/pipeline/test-user.json index 3e17d54c42d..63fc5ab8c4e 100644 --- a/packages/entityanalytics_ad/data_stream/user/_dev/test/pipeline/test-user.json +++ b/packages/entityanalytics_ad/data_stream/user/_dev/test/pipeline/test-user.json @@ -354,6 +354,15 @@ "id": "CN=krbtgt,CN=Users,DC=testserver,DC=local" } }, - {"@timestamp":"2024-03-27T21:30:18.980Z","event":{"action":"completed","end":"2024-03-27T21:30:18.980Z"},"labels":{"identity_source":"entity-analytics-entityanalytics_ad.user-8c3c1f67-428d-4a95-a6de-69a2b8f952c3"}} + { + "@timestamp": "2024-03-27T21:30:18.980Z", + "event": { + "action": "completed", + "end": "2024-03-27T21:30:18.980Z" + }, + "labels": { + "identity_source": "entity-analytics-entityanalytics_ad.user-8c3c1f67-428d-4a95-a6de-69a2b8f952c3" + } + } ] } \ No newline at end of file diff --git a/packages/entityanalytics_ad/data_stream/user/fields/beats.yml b/packages/entityanalytics_ad/data_stream/user/fields/beats.yml index a43f4ff852c..3382e376e77 100644 --- a/packages/entityanalytics_ad/data_stream/user/fields/beats.yml +++ b/packages/entityanalytics_ad/data_stream/user/fields/beats.yml @@ -1,6 +1,3 @@ - name: input.type type: keyword description: Type of filebeat input. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/entityanalytics_ad/data_stream/user/fields/fields.yml b/packages/entityanalytics_ad/data_stream/user/fields/fields.yml index 75a4a0ed1e5..e14a3001d73 100644 --- a/packages/entityanalytics_ad/data_stream/user/fields/fields.yml +++ b/packages/entityanalytics_ad/data_stream/user/fields/fields.yml @@ -119,6 +119,5 @@ type: date - name: when_created type: date - - name: when_changed type: date diff --git a/packages/entityanalytics_ad/docs/README.md b/packages/entityanalytics_ad/docs/README.md index e221b8e3bbe..faf38f3f360 100644 --- a/packages/entityanalytics_ad/docs/README.md +++ b/packages/entityanalytics_ad/docs/README.md @@ -202,7 +202,6 @@ An example event for `user` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of filebeat input. | keyword | | labels.identity_source | | keyword | -| tags | User defined tags. | keyword | | user.account.activated_date | | date | | user.account.change_date | | date | | user.account.create_date | | date | diff --git a/packages/entityanalytics_ad/manifest.yml b/packages/entityanalytics_ad/manifest.yml index 9e4f5f16520..5a0972bcd09 100644 --- a/packages/entityanalytics_ad/manifest.yml +++ b/packages/entityanalytics_ad/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: entityanalytics_ad title: Active Directory Entity Analytics -version: "0.0.1" +version: "0.1.0" description: "Collect User Identities from Active Directory Entity with Elastic Agent." type: integration categories: From d023a5940696dacb87fc51647ffd45c92b67e9d9 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:24:30 +0930 Subject: [PATCH 037/121] [entityanalytics_okta] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/entityanalytics_okta --- packages/entityanalytics_okta/_dev/build/build.yml | 1 - packages/entityanalytics_okta/changelog.yml | 5 +++++ .../entityanalytics_okta/data_stream/user/fields/beats.yml | 3 --- packages/entityanalytics_okta/docs/README.md | 1 - packages/entityanalytics_okta/manifest.yml | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/entityanalytics_okta/_dev/build/build.yml b/packages/entityanalytics_okta/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/entityanalytics_okta/_dev/build/build.yml +++ b/packages/entityanalytics_okta/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/entityanalytics_okta/changelog.yml b/packages/entityanalytics_okta/changelog.yml index ccb00e5a8b7..52ea7b91fd2 100644 --- a/packages/entityanalytics_okta/changelog.yml +++ b/packages/entityanalytics_okta/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.2.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.1.0" changes: - description: Set sensitive values as secret. diff --git a/packages/entityanalytics_okta/data_stream/user/fields/beats.yml b/packages/entityanalytics_okta/data_stream/user/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/entityanalytics_okta/data_stream/user/fields/beats.yml +++ b/packages/entityanalytics_okta/data_stream/user/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/entityanalytics_okta/docs/README.md b/packages/entityanalytics_okta/docs/README.md index c9769b2ba00..9a35f2095dc 100644 --- a/packages/entityanalytics_okta/docs/README.md +++ b/packages/entityanalytics_okta/docs/README.md @@ -324,7 +324,6 @@ An example event for `user` looks as following: | input.type | Type of filebeat input. | keyword | | labels.identity_source | | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | | user.account.activated_date | | date | | user.account.change_date | | date | | user.account.create_date | | date | diff --git a/packages/entityanalytics_okta/manifest.yml b/packages/entityanalytics_okta/manifest.yml index d6f3b28bd8c..e7650a2016d 100644 --- a/packages/entityanalytics_okta/manifest.yml +++ b/packages/entityanalytics_okta/manifest.yml @@ -1,14 +1,14 @@ format_version: "3.0.2" name: entityanalytics_okta title: Okta Entity Analytics -version: "1.1.0" +version: "1.2.0" description: "Collect User Identities from Okta with Elastic Agent." type: integration categories: - security conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" elastic: subscription: "basic" screenshots: From dc8b131bd8f5c3151c486b6228dd39cbcda847c2 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:24:33 +0930 Subject: [PATCH 038/121] [eset_protect] - Updated fields definitions Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/eset_protect --- packages/eset_protect/changelog.yml | 5 +++++ packages/eset_protect/data_stream/detection/fields/beats.yml | 3 --- .../eset_protect/data_stream/device_task/fields/beats.yml | 3 --- packages/eset_protect/data_stream/event/fields/beats.yml | 3 --- packages/eset_protect/docs/README.md | 3 --- packages/eset_protect/manifest.yml | 2 +- 6 files changed, 6 insertions(+), 13 deletions(-) diff --git a/packages/eset_protect/changelog.yml b/packages/eset_protect/changelog.yml index 28c1af82616..792ad666df9 100644 --- a/packages/eset_protect/changelog.yml +++ b/packages/eset_protect/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.1.0" + changes: + - description: Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.0.0" changes: - description: Release package as GA. diff --git a/packages/eset_protect/data_stream/detection/fields/beats.yml b/packages/eset_protect/data_stream/detection/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/eset_protect/data_stream/detection/fields/beats.yml +++ b/packages/eset_protect/data_stream/detection/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/eset_protect/data_stream/device_task/fields/beats.yml b/packages/eset_protect/data_stream/device_task/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/eset_protect/data_stream/device_task/fields/beats.yml +++ b/packages/eset_protect/data_stream/device_task/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/eset_protect/data_stream/event/fields/beats.yml b/packages/eset_protect/data_stream/event/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/eset_protect/data_stream/event/fields/beats.yml +++ b/packages/eset_protect/data_stream/event/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/eset_protect/docs/README.md b/packages/eset_protect/docs/README.md index 9f9d7424bdc..2746af7ce61 100644 --- a/packages/eset_protect/docs/README.md +++ b/packages/eset_protect/docs/README.md @@ -252,7 +252,6 @@ An example event for `detection` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Device Task @@ -386,7 +385,6 @@ An example event for `device_task` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Event @@ -626,5 +624,4 @@ An example event for `event` looks as following: | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | User defined tags. | keyword | diff --git a/packages/eset_protect/manifest.yml b/packages/eset_protect/manifest.yml index 2d45a0dc4e6..b8d9f67c55f 100644 --- a/packages/eset_protect/manifest.yml +++ b/packages/eset_protect/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.3 name: eset_protect title: ESET PROTECT -version: 1.0.0 +version: "1.1.0" description: Collect logs from ESET PROTECT with Elastic Agent. type: integration categories: From 4cadbbe25ed272db9ef13d03fb017e840def2db4 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:24:35 +0930 Subject: [PATCH 039/121] [f5_bigip] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/f5_bigip --- packages/f5_bigip/changelog.yml | 5 + .../f5_bigip/data_stream/log/fields/agent.yml | 147 ------------------ .../f5_bigip/data_stream/log/fields/ecs.yml | 114 -------------- packages/f5_bigip/docs/README.md | 89 ----------- packages/f5_bigip/manifest.yml | 4 +- 5 files changed, 7 insertions(+), 352 deletions(-) delete mode 100644 packages/f5_bigip/data_stream/log/fields/ecs.yml diff --git a/packages/f5_bigip/changelog.yml b/packages/f5_bigip/changelog.yml index b3e2e596495..e88c271b733 100644 --- a/packages/f5_bigip/changelog.yml +++ b/packages/f5_bigip/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.17.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.16.0" changes: - description: Clarify the supported events in README. diff --git a/packages/f5_bigip/data_stream/log/fields/agent.yml b/packages/f5_bigip/data_stream/log/fields/agent.yml index 1740ca457d3..b29a069dffd 100644 --- a/packages/f5_bigip/data_stream/log/fields/agent.yml +++ b/packages/f5_bigip/data_stream/log/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: >- diff --git a/packages/f5_bigip/data_stream/log/fields/ecs.yml b/packages/f5_bigip/data_stream/log/fields/ecs.yml deleted file mode 100644 index a51a91ad790..00000000000 --- a/packages/f5_bigip/data_stream/log/fields/ecs.yml +++ /dev/null @@ -1,114 +0,0 @@ -- external: ecs - name: client.ip -- external: ecs - name: destination.domain -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: dns.question.name -- external: ecs - name: dns.question.type -- external: ecs - name: ecs.version -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.severity -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: host.geo.country_iso_code -- external: ecs - name: host.geo.continent_name -- external: ecs - name: host.geo.country_name -- external: ecs - name: http.request.method -- external: ecs - name: http.request.referrer -- external: ecs - name: http.response.status_code -- external: ecs - name: http.version -- external: ecs - name: log.file.path -- external: ecs - name: log.level -- external: ecs - name: log.syslog.severity.code -- external: ecs - name: network.application -- external: ecs - name: network.bytes -- external: ecs - name: network.direction -- external: ecs - name: network.protocol -- external: ecs - name: network.transport -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: server.ip -- external: ecs - name: source.domain -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: source.user.group.name -- external: ecs - name: source.user.name -- external: ecs - name: tags -- external: ecs - name: url.domain -- external: ecs - name: url.extension -- external: ecs - name: url.fragment -- external: ecs - name: url.original -- external: ecs - name: url.password -- external: ecs - name: url.path -- external: ecs - name: url.port -- external: ecs - name: url.query -- external: ecs - name: url.scheme -- external: ecs - name: url.username -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version -- external: ecs - name: user.name diff --git a/packages/f5_bigip/docs/README.md b/packages/f5_bigip/docs/README.md index f9719439fa4..ba4110018b8 100644 --- a/packages/f5_bigip/docs/README.md +++ b/packages/f5_bigip/docs/README.md @@ -360,38 +360,12 @@ An example event for `log` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| dns.question.name | The name being queried. If the name field contains non-printable characters (below 32 or above 126), those characters should be represented as escaped base 10 integers (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, and line feeds should be converted to \t, \r, and \n respectively. | keyword | -| dns.question.type | The type of record being queried. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | f5_bigip.log.abandoned_conns | | long | | f5_bigip.log.accept_fails | | long | | f5_bigip.log.accepts | | long | @@ -683,78 +657,15 @@ An example event for `log` looks as following: | f5_bigip.log.websocket.message_type | | keyword | | f5_bigip.log.wl_events | | long | | f5_bigip.log.x_forwarded_for_header_value | | ip | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.geo.continent_name | Name of the continent. | keyword | -| host.geo.country_iso_code | Country ISO code. | keyword | -| host.geo.country_name | Country name. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.request.referrer | Referrer for this HTTP request. | keyword | -| http.response.status_code | HTTP response status code. | long | -| http.version | HTTP version. | keyword | | input.type | Input type | keyword | | log.file.device_id | ID of the device containing the filesystem where the file resides. | keyword | | log.file.fingerprint | The sha256 fingerprint identity of the file when fingerprinting is enabled. | keyword | | log.file.idxhi | The high-order part of a unique identifier that is associated with a file. (Windows-only) | keyword | | log.file.idxlo | The low-order part of a unique identifier that is associated with a file. (Windows-only) | keyword | | log.file.inode | Inode number of the log file. | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.file.vol | The serial number of the volume that contains a file. (Windows-only) | keyword | -| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | | log.offset | Log offset | long | -| log.syslog.severity.code | The Syslog numeric severity of the log event, if available. If the event source publishing via Syslog provides a different numeric severity value (e.g. firewall, IDS), your source's numeric severity should go to `event.severity`. If the event source does not specify a distinct severity, you can optionally copy the Syslog severity to `event.severity`. | long | -| network.application | When a specific application or service is identified from network connection details (source/dest IPs, ports, certificates, or wire format), this field captures the application's or service's name. For example, the original event identifies the network connection being from a specific web service in a `https` network connection, like `facebook` or `twitter`. The field value must be normalized to lowercase for querying. | keyword | -| network.bytes | Total bytes transferred in both directions. If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. | long | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| server.ip | IP address of the server (IPv4 or IPv6). | ip | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| source.user.group.name | Name of the group. | keyword | -| source.user.name | Short name or login of the user. | keyword | -| source.user.name.text | Multi-field of `source.user.name`. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.password | Password of the request. | keyword | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.username | Username of the request. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | diff --git a/packages/f5_bigip/manifest.yml b/packages/f5_bigip/manifest.yml index c98266e7225..e4626bfb335 100644 --- a/packages/f5_bigip/manifest.yml +++ b/packages/f5_bigip/manifest.yml @@ -1,14 +1,14 @@ format_version: "3.0.2" name: f5_bigip title: F5 BIG-IP -version: "1.16.0" +version: "1.17.0" description: Collect logs from F5 BIG-IP with Elastic Agent. type: integration categories: - security conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: subscription: basic screenshots: From e7c2b0d84cc9943d73692f21932fb92c6d452b6e Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:24:36 +0930 Subject: [PATCH 040/121] [fireeye] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^7.16.0 || ^8.0.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/fireeye --- packages/fireeye/changelog.yml | 5 + .../fireeye/data_stream/nx/fields/agent.yml | 143 ---------------- .../fireeye/data_stream/nx/fields/ecs.yml | 156 ------------------ packages/fireeye/docs/README.md | 110 ------------ packages/fireeye/manifest.yml | 4 +- 5 files changed, 7 insertions(+), 411 deletions(-) delete mode 100644 packages/fireeye/data_stream/nx/fields/ecs.yml diff --git a/packages/fireeye/changelog.yml b/packages/fireeye/changelog.yml index ed0e6d63b36..682e74ca1dd 100644 --- a/packages/fireeye/changelog.yml +++ b/packages/fireeye/changelog.yml @@ -1,3 +1,8 @@ +- version: "1.23.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.22.0" changes: - description: Update manifest format version to v3.0.3. diff --git a/packages/fireeye/data_stream/nx/fields/agent.yml b/packages/fireeye/data_stream/nx/fields/agent.yml index 368be734273..ae4fc1ddd0c 100644 --- a/packages/fireeye/data_stream/nx/fields/agent.yml +++ b/packages/fireeye/data_stream/nx/fields/agent.yml @@ -5,158 +5,15 @@ footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." - - name: id - level: core - type: keyword - ignore_above: 1024 - description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." - name: containerized type: boolean description: > diff --git a/packages/fireeye/data_stream/nx/fields/ecs.yml b/packages/fireeye/data_stream/nx/fields/ecs.yml deleted file mode 100644 index fa8df0acf96..00000000000 --- a/packages/fireeye/data_stream/nx/fields/ecs.yml +++ /dev/null @@ -1,156 +0,0 @@ -- external: ecs - name: destination.domain -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: destination.bytes -- external: ecs - name: destination.packets -- external: ecs - name: ecs.version -- external: ecs - name: host.ip -- external: ecs - name: http.request.method -- external: ecs - name: http.request.referrer -- external: ecs - name: http.response.body.bytes -- external: ecs - name: http.response.status_code -- external: ecs - name: http.version -- external: ecs - name: http.request.mime_type -- external: ecs - name: http.response.bytes -- external: ecs - name: log.file.path -- external: ecs - name: related.ip -- external: ecs - name: related.hash -- external: ecs - name: source.bytes -- external: ecs - name: source.packets -- external: ecs - name: source.address -- external: ecs - name: source.port -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: destination.address -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: tags -- external: ecs - name: url.domain -- external: ecs - name: url.extension -- external: ecs - name: url.fragment -- external: ecs - name: url.original -- external: ecs - name: url.path -- external: ecs - name: url.scheme -- external: ecs - name: user.name -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version -- external: ecs - name: network.transport -- external: ecs - name: network.protocol -- external: ecs - name: network.community_id -- external: ecs - name: network.iana_number -- external: ecs - name: event.type -- external: ecs - name: observer.ingress.interface.name -- external: ecs - name: dns.response_code -- external: ecs - name: dns.question.name -- external: ecs - name: dns.question.type -- external: ecs - name: dns.answers.ttl -- external: ecs - name: dns.type -- external: ecs - name: dns.id -- external: ecs - name: tls.client.issuer -- external: ecs - name: tls.client.ja3 -- external: ecs - name: tls.client.not_before -- external: ecs - name: tls.client.not_after -- external: ecs - name: tls.client.server_name -- external: ecs - name: tls.client.subject -- external: ecs - name: tls.server.ja3s -- external: ecs - name: tls.version -- external: ecs - name: observer.product -- external: ecs - name: observer.vendor diff --git a/packages/fireeye/docs/README.md b/packages/fireeye/docs/README.md index efea6222503..bd7c2de556d 100644 --- a/packages/fireeye/docs/README.md +++ b/packages/fireeye/docs/README.md @@ -17,48 +17,12 @@ The `nx` integration ingests network security logs from FireEye NX through TCP/U | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.bytes | Bytes sent from the destination to the source. | long | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.packets | Packets sent from the destination to the source. | long | -| destination.port | Port of the destination. | long | -| dns.answers.ttl | The time interval in seconds that this resource record may be cached before it should be discarded. Zero values mean that the data should not be cached. | long | -| dns.id | The DNS packet identifier assigned by the program that generated the query. The identifier is copied to the response. | keyword | -| dns.question.name | The name being queried. If the name field contains non-printable characters (below 32 or above 126), those characters should be represented as escaped base 10 integers (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, and line feeds should be converted to \t, \r, and \n respectively. | keyword | -| dns.question.type | The type of record being queried. | keyword | -| dns.response_code | The DNS response code. | keyword | -| dns.type | The type of DNS event captured, query or answer. If your source of DNS events only gives you DNS queries, you should only create dns events of type `dns.type:query`. If your source of DNS events gives you answers as well, you should create one event per query (optionally as soon as the query is seen). And a second event containing all query details as well as an array of answers. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | event.dataset | Event dataset | constant_keyword | | event.module | Event module | constant_keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | fireeye.nx.fileinfo.filename | File name. | keyword | | fireeye.nx.fileinfo.magic | Fileinfo magic. | keyword | | fireeye.nx.fileinfo.md5 | File hash. | keyword | @@ -79,94 +43,20 @@ The `nx` integration ingests network security logs from FireEye NX through TCP/U | fireeye.nx.tcp.tcp_flags | TCP flags. | keyword | | fireeye.nx.tcp.tcp_flags_tc | TCP flags. | keyword | | fireeye.nx.tcp.tcp_flags_ts | TCP flags. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.request.mime_type | Mime type of the body of the request. This value must only be populated based on the content of the request body, not on the `Content-Type` header. Comparing the mime type of a request with the request's Content-Type header can be helpful in detecting threats or misconfigured clients. | keyword | -| http.request.referrer | Referrer for this HTTP request. | keyword | -| http.response.body.bytes | Size in bytes of the response body. | long | -| http.response.bytes | Total size in bytes of the response (body and headers). | long | -| http.response.status_code | HTTP response status code. | long | -| http.version | HTTP version. | keyword | | input.type | Input type | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | Log offset | long | | log.source.address | Logs Source Raw address. | keyword | -| network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword | -| network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| observer.ingress.interface.name | Interface name as reported by the system. | keyword | -| observer.product | The product name of the observer. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.packets | Packets sent from the source to the destination. | long | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | | tls.client.ciphersuites | TLS cipher suites by client. | long | | tls.client.fingerprint | TLS fingerprint. | keyword | -| tls.client.issuer | Distinguished name of subject of the issuer of the x.509 certificate presented by the client. | keyword | -| tls.client.ja3 | A hash that identifies clients based on how they perform an SSL/TLS handshake. | keyword | | tls.client.ja3_string | A hash that identifies clients based on how they perform an SSL/TLS handshake. | keyword | -| tls.client.not_after | Date/Time indicating when client certificate is no longer considered valid. | date | -| tls.client.not_before | Date/Time indicating when client certificate is first considered valid. | date | -| tls.client.server_name | Also called an SNI, this tells the server which hostname to which the client is attempting to connect to. When this value is available, it should get copied to `destination.domain`. | keyword | -| tls.client.subject | Distinguished name of subject of the x.509 certificate presented by the client. | keyword | | tls.client.tls_exts | TLS extensions set by client. | long | | tls.public_keylength | TLS public key length. | long | | tls.server.ciphersuite | TLS cipher suites by server. | long | -| tls.server.ja3s | A hash that identifies servers based on how they perform an SSL/TLS handshake. | keyword | | tls.server.ja3s_string | A hash that identifies servers based on how they perform an SSL/TLS handshake. | keyword | | tls.server.tls_exts | TLS extensions set by server. | long | -| tls.version | Numeric part of the version parsed from the original string. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | An example event for `nx` looks as following: diff --git a/packages/fireeye/manifest.yml b/packages/fireeye/manifest.yml index 3c2e746df22..fbb822585d5 100644 --- a/packages/fireeye/manifest.yml +++ b/packages/fireeye/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: fireeye title: "FireEye Network Security" -version: "1.22.0" +version: "1.23.0" description: Collect logs from FireEye NX with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - security conditions: kibana: - version: "^7.16.0 || ^8.0.0" + version: "^8.13.0" icons: - src: /img/FireEye-logo.svg title: Fireeye logo From fae0200dbeec73ec435d4974fab7a145e4e215b1 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:24:37 +0930 Subject: [PATCH 041/121] [forcepoint_web] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.5.1 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/forcepoint_web --- packages/forcepoint_web/changelog.yml | 5 + .../data_stream/logs/fields/ecs.yml | 220 ------------------ packages/forcepoint_web/docs/README.md | 117 ---------- packages/forcepoint_web/manifest.yml | 4 +- 4 files changed, 7 insertions(+), 339 deletions(-) diff --git a/packages/forcepoint_web/changelog.yml b/packages/forcepoint_web/changelog.yml index 77378378654..0c9f100fca8 100644 --- a/packages/forcepoint_web/changelog.yml +++ b/packages/forcepoint_web/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.9.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.8.0" changes: - description: Upgrade to package spec 3.0.3. diff --git a/packages/forcepoint_web/data_stream/logs/fields/ecs.yml b/packages/forcepoint_web/data_stream/logs/fields/ecs.yml index e5dedbb37b7..adb0dc85322 100644 --- a/packages/forcepoint_web/data_stream/logs/fields/ecs.yml +++ b/packages/forcepoint_web/data_stream/logs/fields/ecs.yml @@ -1,222 +1,2 @@ - external: ecs name: '@timestamp' -- external: ecs - name: container.id -- external: ecs - name: destination.address -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.bytes -- external: ecs - name: destination.domain -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.geo.name -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.ip -- external: ecs - name: destination.nat.ip -- external: ecs - name: destination.nat.port -- external: ecs - name: destination.packets -- external: ecs - name: destination.port -- external: ecs - name: destination.user.email -- external: ecs - name: destination.user.name -- external: ecs - name: ecs.version -- external: ecs - name: error.code -- external: ecs - name: error.message -- external: ecs - name: event.category -- external: ecs - name: event.code -- external: ecs - name: event.dataset -- external: ecs - name: event.duration -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.module -- external: ecs - name: event.outcome -- external: ecs - name: event.reference -- external: ecs - name: event.start -- external: ecs - name: event.timezone -- external: ecs - name: event.type -- external: ecs - name: file.extension -- external: ecs - name: file.name -- external: ecs - name: file.size -- external: ecs - name: http.request.method -- external: ecs - name: http.response.status_code -- external: ecs - name: log.level -- external: ecs - name: message -- external: ecs - name: network.application -- external: ecs - name: network.bytes -- external: ecs - name: network.direction -- external: ecs - name: network.iana_number -- external: ecs - name: network.transport -- external: ecs - name: network.packets -- external: ecs - name: network.protocol -- external: ecs - name: observer.egress.interface.name -- external: ecs - name: observer.ingress.interface.name -- external: ecs - name: observer.name -- external: ecs - name: observer.product -- external: ecs - name: observer.serial_number -- external: ecs - name: observer.type -- external: ecs - name: observer.vendor -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: rule.category -- external: ecs - name: rule.description -- external: ecs - name: rule.id -- external: ecs - name: rule.name -- external: ecs - name: rule.ruleset -- external: ecs - name: rule.uuid -- external: ecs - name: source.address -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.bytes -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.name -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: source.mac -- external: ecs - name: source.nat.ip -- external: ecs - name: source.nat.port -- external: ecs - name: source.packets -- external: ecs - name: source.port -- external: ecs - name: source.user.email -- external: ecs - name: source.user.group.name -- external: ecs - name: source.user.name -- external: ecs - name: tags -- external: ecs - name: tls.client.issuer -- external: ecs - name: tls.client.server_name -- external: ecs - name: tls.client.x509.issuer.common_name -- external: ecs - name: tls.server.issuer -- external: ecs - name: tls.server.x509.issuer.common_name -- external: ecs - name: tls.server.x509.subject.common_name -- external: ecs - name: url.domain -- external: ecs - name: url.path -- external: ecs - name: url.original -- external: ecs - name: url.port -- external: ecs - name: url.registered_domain -- external: ecs - name: url.scheme -- external: ecs - name: url.subdomain -- external: ecs - name: url.top_level_domain -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.version -- external: ecs - name: vulnerability.category diff --git a/packages/forcepoint_web/docs/README.md b/packages/forcepoint_web/docs/README.md index eef17c16676..a7b7906261b 100644 --- a/packages/forcepoint_web/docs/README.md +++ b/packages/forcepoint_web/docs/README.md @@ -207,50 +207,9 @@ The following fields may be used by the package: | Field | Description | Type | |---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | -| container.id | Unique container id. | keyword | | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.bytes | Bytes sent from the destination to the source. | long | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.nat.ip | Translated ip of destination based NAT sessions (e.g. internet to private DMZ) Typically used with load balancers, firewalls, or routers. | ip | -| destination.nat.port | Port the source session is translated to by NAT Device. Typically used with load balancers, firewalls, or routers. | long | -| destination.packets | Packets sent from the destination to the source. | long | -| destination.port | Port of the destination. | long | -| destination.user.email | User email address. | keyword | -| destination.user.name | Short name or login of the user. | keyword | -| destination.user.name.text | Multi-field of `destination.user.name`. | match_only_text | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.code | Error code describing the error. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | -| event.duration | Duration of the event in nanoseconds. If `event.start` and `event.end` are known this value should be the difference between the end and start time. | long | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.reference | Reference URL linking to additional information about this event. This URL links to a static definition of this event. Alert events, indicated by `event.kind:alert`, are a common use case for this field. | keyword | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | | forcepoint_web.action | | keyword | | forcepoint_web.category | | keyword | | forcepoint_web.connection_ip | | keyword | @@ -266,81 +225,5 @@ The following fields may be used by the package: | forcepoint_web.user | | keyword | | forcepoint_web.user_agent_string | | keyword | | forcepoint_web.workstation | | keyword | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.response.status_code | HTTP response status code. | long | | input.type | | keyword | -| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.application | When a specific application or service is identified from network connection details (source/dest IPs, ports, certificates, or wire format), this field captures the application's or service's name. For example, the original event identifies the network connection being from a specific web service in a `https` network connection, like `facebook` or `twitter`. The field value must be normalized to lowercase for querying. | keyword | -| network.bytes | Total bytes transferred in both directions. If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. | long | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword | -| network.packets | Total packets transferred in both directions. If `source.packets` and `destination.packets` are known, `network.packets` is their sum. | long | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| observer.egress.interface.name | Interface name as reported by the system. | keyword | -| observer.ingress.interface.name | Interface name as reported by the system. | keyword | -| observer.name | Custom name of the observer. This is a name that can be given to an observer. This can be helpful for example if multiple firewalls of the same model are used in an organization. If no custom name is needed, the field can be left empty. | keyword | -| observer.product | The product name of the observer. | keyword | -| observer.serial_number | Observer serial number. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.category | A categorization value keyword used by the entity using the rule for detection of this event. | keyword | -| rule.description | The description of the rule generating the event. | keyword | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| rule.ruleset | Name of the ruleset, policy, group, or parent category in which the rule used to generate this event is a member. | keyword | -| rule.uuid | A rule ID that is unique within the scope of a set or group of agents, observers, or other entities using the rule for detection of this event. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.mac | MAC address of the source. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| source.nat.ip | Translated ip of source based NAT sessions (e.g. internal client to internet) Typically connections traversing load balancers, firewalls, or routers. | ip | -| source.nat.port | Translated port of source based NAT sessions. (e.g. internal client to internet) Typically used with load balancers, firewalls, or routers. | long | -| source.packets | Packets sent from the source to the destination. | long | -| source.port | Port of the source. | long | -| source.user.email | User email address. | keyword | -| source.user.group.name | Name of the group. | keyword | -| source.user.name | Short name or login of the user. | keyword | -| source.user.name.text | Multi-field of `source.user.name`. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| tls.client.issuer | Distinguished name of subject of the issuer of the x.509 certificate presented by the client. | keyword | -| tls.client.server_name | Also called an SNI, this tells the server which hostname to which the client is attempting to connect to. When this value is available, it should get copied to `destination.domain`. | keyword | -| tls.client.x509.issuer.common_name | List of common name (CN) of issuing certificate authority. | keyword | -| tls.server.issuer | Subject of the issuer of the x.509 certificate presented by the server. | keyword | -| tls.server.x509.issuer.common_name | List of common name (CN) of issuing certificate authority. | keyword | -| tls.server.x509.subject.common_name | List of common names (CN) of subject. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | -| url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.version | Version of the user agent. | keyword | -| vulnerability.category | The type of system or architecture that the vulnerability affects. These may be platform-specific (for example, Debian or SUSE) or general (for example, Database or Firewall). For example (https://qualysguard.qualys.com/qwebhelp/fo_portal/knowledgebase/vulnerability_categories.htm[Qualys vulnerability categories]) This field must be an array. | keyword | diff --git a/packages/forcepoint_web/manifest.yml b/packages/forcepoint_web/manifest.yml index a1a653ccaf5..3af8f057913 100644 --- a/packages/forcepoint_web/manifest.yml +++ b/packages/forcepoint_web/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: forcepoint_web title: "Forcepoint Web Security" -version: "1.8.0" +version: "1.9.0" source: license: "Elastic-2.0" description: "Forcepoint Web Security" @@ -11,7 +11,7 @@ categories: - security conditions: kibana: - version: "^8.5.1" + version: "^8.13.0" elastic: subscription: "basic" screenshots: From 6de5352ef35496246d871357b9e70ff82acb537b Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:24:44 +0930 Subject: [PATCH 042/121] [forgerock] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/forgerock --- packages/forgerock/changelog.yml | 5 + .../am_access/fields/base-fields.yml | 6 - .../data_stream/am_access/fields/ecs.yml | 38 ------ .../am_activity/fields/base-fields.yml | 6 - .../data_stream/am_activity/fields/ecs.yml | 20 --- .../am_authentication/fields/base-fields.yml | 6 - .../am_authentication/fields/ecs.yml | 16 --- .../am_config/fields/base-fields.yml | 6 - .../data_stream/am_config/fields/ecs.yml | 18 --- .../am_core/fields/base-fields.yml | 6 - .../data_stream/am_core/fields/ecs.yml | 18 --- .../idm_access/fields/base-fields.yml | 6 - .../data_stream/idm_access/fields/ecs.yml | 20 --- .../idm_activity/fields/base-fields.yml | 6 - .../data_stream/idm_activity/fields/ecs.yml | 14 -- .../idm_authentication/fields/base-fields.yml | 6 - .../idm_authentication/fields/ecs.yml | 12 -- .../idm_config/fields/base-fields.yml | 6 - .../data_stream/idm_config/fields/ecs.yml | 14 -- .../idm_core/fields/base-fields.yml | 6 - .../data_stream/idm_core/fields/ecs.yml | 6 - .../idm_sync/fields/base-fields.yml | 6 - .../data_stream/idm_sync/fields/ecs.yml | 14 -- packages/forgerock/docs/README.md | 120 ------------------ packages/forgerock/manifest.yml | 4 +- 25 files changed, 7 insertions(+), 378 deletions(-) delete mode 100644 packages/forgerock/data_stream/am_access/fields/ecs.yml delete mode 100644 packages/forgerock/data_stream/am_activity/fields/ecs.yml delete mode 100644 packages/forgerock/data_stream/am_authentication/fields/ecs.yml delete mode 100644 packages/forgerock/data_stream/am_config/fields/ecs.yml delete mode 100644 packages/forgerock/data_stream/am_core/fields/ecs.yml delete mode 100644 packages/forgerock/data_stream/idm_access/fields/ecs.yml delete mode 100644 packages/forgerock/data_stream/idm_activity/fields/ecs.yml delete mode 100644 packages/forgerock/data_stream/idm_authentication/fields/ecs.yml delete mode 100644 packages/forgerock/data_stream/idm_config/fields/ecs.yml delete mode 100644 packages/forgerock/data_stream/idm_core/fields/ecs.yml delete mode 100644 packages/forgerock/data_stream/idm_sync/fields/ecs.yml diff --git a/packages/forgerock/changelog.yml b/packages/forgerock/changelog.yml index 1758e86058a..9e04fb0aa75 100644 --- a/packages/forgerock/changelog.yml +++ b/packages/forgerock/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.18.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.17.1" changes: - description: Fix sample event. diff --git a/packages/forgerock/data_stream/am_access/fields/base-fields.yml b/packages/forgerock/data_stream/am_access/fields/base-fields.yml index 5c15b940850..0f62ee563d1 100644 --- a/packages/forgerock/data_stream/am_access/fields/base-fields.yml +++ b/packages/forgerock/data_stream/am_access/fields/base-fields.yml @@ -4,12 +4,6 @@ external: ecs - name: data_stream.namespace external: ecs -- name: event.module - external: ecs - value: forgerock -- name: event.dataset - external: ecs - value: forgerock.audit - name: '@timestamp' external: ecs - name: input.type diff --git a/packages/forgerock/data_stream/am_access/fields/ecs.yml b/packages/forgerock/data_stream/am_access/fields/ecs.yml deleted file mode 100644 index 1d8c46e6ed1..00000000000 --- a/packages/forgerock/data_stream/am_access/fields/ecs.yml +++ /dev/null @@ -1,38 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.id -- external: ecs - name: transaction.id -- external: ecs - name: user.id -- external: ecs - name: client.ip -- external: ecs - name: client.port -- external: ecs - name: client.domain -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: event.action -- external: ecs - name: http.request.method -- external: ecs - name: event.outcome -- external: ecs - name: http.response.status_code -- external: ecs - name: http.response.body.content -- external: ecs - name: event.duration -- external: ecs - name: server.ip -- external: ecs - name: service.name -- external: ecs - name: observer.vendor -- external: ecs - name: tags diff --git a/packages/forgerock/data_stream/am_activity/fields/base-fields.yml b/packages/forgerock/data_stream/am_activity/fields/base-fields.yml index 5c15b940850..0f62ee563d1 100644 --- a/packages/forgerock/data_stream/am_activity/fields/base-fields.yml +++ b/packages/forgerock/data_stream/am_activity/fields/base-fields.yml @@ -4,12 +4,6 @@ external: ecs - name: data_stream.namespace external: ecs -- name: event.module - external: ecs - value: forgerock -- name: event.dataset - external: ecs - value: forgerock.audit - name: '@timestamp' external: ecs - name: input.type diff --git a/packages/forgerock/data_stream/am_activity/fields/ecs.yml b/packages/forgerock/data_stream/am_activity/fields/ecs.yml deleted file mode 100644 index 1e7792dc5b2..00000000000 --- a/packages/forgerock/data_stream/am_activity/fields/ecs.yml +++ /dev/null @@ -1,20 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.id -- external: ecs - name: transaction.id -- external: ecs - name: user.id -- external: ecs - name: tags -- external: ecs - name: observer.vendor -- external: ecs - name: event.action -- external: ecs - name: event.duration -- external: ecs - name: service.name -- external: ecs - name: user.effective.id diff --git a/packages/forgerock/data_stream/am_authentication/fields/base-fields.yml b/packages/forgerock/data_stream/am_authentication/fields/base-fields.yml index 5c15b940850..0f62ee563d1 100644 --- a/packages/forgerock/data_stream/am_authentication/fields/base-fields.yml +++ b/packages/forgerock/data_stream/am_authentication/fields/base-fields.yml @@ -4,12 +4,6 @@ external: ecs - name: data_stream.namespace external: ecs -- name: event.module - external: ecs - value: forgerock -- name: event.dataset - external: ecs - value: forgerock.audit - name: '@timestamp' external: ecs - name: input.type diff --git a/packages/forgerock/data_stream/am_authentication/fields/ecs.yml b/packages/forgerock/data_stream/am_authentication/fields/ecs.yml deleted file mode 100644 index 5c532e03dff..00000000000 --- a/packages/forgerock/data_stream/am_authentication/fields/ecs.yml +++ /dev/null @@ -1,16 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.id -- external: ecs - name: transaction.id -- external: ecs - name: user.id -- external: ecs - name: event.outcome -- external: ecs - name: observer.vendor -- external: ecs - name: tags -- external: ecs - name: service.name diff --git a/packages/forgerock/data_stream/am_config/fields/base-fields.yml b/packages/forgerock/data_stream/am_config/fields/base-fields.yml index 5c15b940850..0f62ee563d1 100644 --- a/packages/forgerock/data_stream/am_config/fields/base-fields.yml +++ b/packages/forgerock/data_stream/am_config/fields/base-fields.yml @@ -4,12 +4,6 @@ external: ecs - name: data_stream.namespace external: ecs -- name: event.module - external: ecs - value: forgerock -- name: event.dataset - external: ecs - value: forgerock.audit - name: '@timestamp' external: ecs - name: input.type diff --git a/packages/forgerock/data_stream/am_config/fields/ecs.yml b/packages/forgerock/data_stream/am_config/fields/ecs.yml deleted file mode 100644 index 656f6403ffe..00000000000 --- a/packages/forgerock/data_stream/am_config/fields/ecs.yml +++ /dev/null @@ -1,18 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.id -- external: ecs - name: transaction.id -- external: ecs - name: user.effective.id -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: observer.vendor -- external: ecs - name: tags -- external: ecs - name: user.id diff --git a/packages/forgerock/data_stream/am_core/fields/base-fields.yml b/packages/forgerock/data_stream/am_core/fields/base-fields.yml index 5c15b940850..0f62ee563d1 100644 --- a/packages/forgerock/data_stream/am_core/fields/base-fields.yml +++ b/packages/forgerock/data_stream/am_core/fields/base-fields.yml @@ -4,12 +4,6 @@ external: ecs - name: data_stream.namespace external: ecs -- name: event.module - external: ecs - value: forgerock -- name: event.dataset - external: ecs - value: forgerock.audit - name: '@timestamp' external: ecs - name: input.type diff --git a/packages/forgerock/data_stream/am_core/fields/ecs.yml b/packages/forgerock/data_stream/am_core/fields/ecs.yml deleted file mode 100644 index 878f80a1ed3..00000000000 --- a/packages/forgerock/data_stream/am_core/fields/ecs.yml +++ /dev/null @@ -1,18 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: transaction.id -- external: ecs - name: event.reason -- external: ecs - name: log.level -- external: ecs - name: log.logger -- external: ecs - name: process.name -- external: ecs - name: observer.vendor -- external: ecs - name: error.stack_trace -- external: ecs - name: tags diff --git a/packages/forgerock/data_stream/idm_access/fields/base-fields.yml b/packages/forgerock/data_stream/idm_access/fields/base-fields.yml index 5c15b940850..0f62ee563d1 100644 --- a/packages/forgerock/data_stream/idm_access/fields/base-fields.yml +++ b/packages/forgerock/data_stream/idm_access/fields/base-fields.yml @@ -4,12 +4,6 @@ external: ecs - name: data_stream.namespace external: ecs -- name: event.module - external: ecs - value: forgerock -- name: event.dataset - external: ecs - value: forgerock.audit - name: '@timestamp' external: ecs - name: input.type diff --git a/packages/forgerock/data_stream/idm_access/fields/ecs.yml b/packages/forgerock/data_stream/idm_access/fields/ecs.yml deleted file mode 100644 index ee115549390..00000000000 --- a/packages/forgerock/data_stream/idm_access/fields/ecs.yml +++ /dev/null @@ -1,20 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: client.ip -- external: ecs - name: client.port -- external: ecs - name: transaction.id -- external: ecs - name: user.id -- external: ecs - name: server.ip -- external: ecs - name: http.request.method -- external: ecs - name: http.response.status_code -- external: ecs - name: observer.vendor -- external: ecs - name: tags diff --git a/packages/forgerock/data_stream/idm_activity/fields/base-fields.yml b/packages/forgerock/data_stream/idm_activity/fields/base-fields.yml index 5c15b940850..0f62ee563d1 100644 --- a/packages/forgerock/data_stream/idm_activity/fields/base-fields.yml +++ b/packages/forgerock/data_stream/idm_activity/fields/base-fields.yml @@ -4,12 +4,6 @@ external: ecs - name: data_stream.namespace external: ecs -- name: event.module - external: ecs - value: forgerock -- name: event.dataset - external: ecs - value: forgerock.audit - name: '@timestamp' external: ecs - name: input.type diff --git a/packages/forgerock/data_stream/idm_activity/fields/ecs.yml b/packages/forgerock/data_stream/idm_activity/fields/ecs.yml deleted file mode 100644 index db971e84e88..00000000000 --- a/packages/forgerock/data_stream/idm_activity/fields/ecs.yml +++ /dev/null @@ -1,14 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.id -- external: ecs - name: transaction.id -- external: ecs - name: observer.vendor -- external: ecs - name: tags -- external: ecs - name: user.effective.id -- external: ecs - name: user.id diff --git a/packages/forgerock/data_stream/idm_authentication/fields/base-fields.yml b/packages/forgerock/data_stream/idm_authentication/fields/base-fields.yml index 5c15b940850..0f62ee563d1 100644 --- a/packages/forgerock/data_stream/idm_authentication/fields/base-fields.yml +++ b/packages/forgerock/data_stream/idm_authentication/fields/base-fields.yml @@ -4,12 +4,6 @@ external: ecs - name: data_stream.namespace external: ecs -- name: event.module - external: ecs - value: forgerock -- name: event.dataset - external: ecs - value: forgerock.audit - name: '@timestamp' external: ecs - name: input.type diff --git a/packages/forgerock/data_stream/idm_authentication/fields/ecs.yml b/packages/forgerock/data_stream/idm_authentication/fields/ecs.yml deleted file mode 100644 index 01b02b19d6e..00000000000 --- a/packages/forgerock/data_stream/idm_authentication/fields/ecs.yml +++ /dev/null @@ -1,12 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.id -- external: ecs - name: transaction.id -- external: ecs - name: observer.vendor -- external: ecs - name: tags -- external: ecs - name: user.id diff --git a/packages/forgerock/data_stream/idm_config/fields/base-fields.yml b/packages/forgerock/data_stream/idm_config/fields/base-fields.yml index 5c15b940850..0f62ee563d1 100644 --- a/packages/forgerock/data_stream/idm_config/fields/base-fields.yml +++ b/packages/forgerock/data_stream/idm_config/fields/base-fields.yml @@ -4,12 +4,6 @@ external: ecs - name: data_stream.namespace external: ecs -- name: event.module - external: ecs - value: forgerock -- name: event.dataset - external: ecs - value: forgerock.audit - name: '@timestamp' external: ecs - name: input.type diff --git a/packages/forgerock/data_stream/idm_config/fields/ecs.yml b/packages/forgerock/data_stream/idm_config/fields/ecs.yml deleted file mode 100644 index 34a5570d138..00000000000 --- a/packages/forgerock/data_stream/idm_config/fields/ecs.yml +++ /dev/null @@ -1,14 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.id -- external: ecs - name: transaction.id -- external: ecs - name: user.id -- external: ecs - name: user.effective.id -- external: ecs - name: observer.vendor -- external: ecs - name: tags diff --git a/packages/forgerock/data_stream/idm_core/fields/base-fields.yml b/packages/forgerock/data_stream/idm_core/fields/base-fields.yml index 5c15b940850..0f62ee563d1 100644 --- a/packages/forgerock/data_stream/idm_core/fields/base-fields.yml +++ b/packages/forgerock/data_stream/idm_core/fields/base-fields.yml @@ -4,12 +4,6 @@ external: ecs - name: data_stream.namespace external: ecs -- name: event.module - external: ecs - value: forgerock -- name: event.dataset - external: ecs - value: forgerock.audit - name: '@timestamp' external: ecs - name: input.type diff --git a/packages/forgerock/data_stream/idm_core/fields/ecs.yml b/packages/forgerock/data_stream/idm_core/fields/ecs.yml deleted file mode 100644 index 3adc3e10518..00000000000 --- a/packages/forgerock/data_stream/idm_core/fields/ecs.yml +++ /dev/null @@ -1,6 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: observer.vendor -- external: ecs - name: tags diff --git a/packages/forgerock/data_stream/idm_sync/fields/base-fields.yml b/packages/forgerock/data_stream/idm_sync/fields/base-fields.yml index 5c15b940850..0f62ee563d1 100644 --- a/packages/forgerock/data_stream/idm_sync/fields/base-fields.yml +++ b/packages/forgerock/data_stream/idm_sync/fields/base-fields.yml @@ -4,12 +4,6 @@ external: ecs - name: data_stream.namespace external: ecs -- name: event.module - external: ecs - value: forgerock -- name: event.dataset - external: ecs - value: forgerock.audit - name: '@timestamp' external: ecs - name: input.type diff --git a/packages/forgerock/data_stream/idm_sync/fields/ecs.yml b/packages/forgerock/data_stream/idm_sync/fields/ecs.yml deleted file mode 100644 index e5a6d114b6d..00000000000 --- a/packages/forgerock/data_stream/idm_sync/fields/ecs.yml +++ /dev/null @@ -1,14 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.id -- external: ecs - name: transaction.id -- external: ecs - name: user.id -- external: ecs - name: event.outcome -- external: ecs - name: observer.vendor -- external: ecs - name: tags diff --git a/packages/forgerock/docs/README.md b/packages/forgerock/docs/README.md index a38ec8d2116..a55cbb1593c 100644 --- a/packages/forgerock/docs/README.md +++ b/packages/forgerock/docs/README.md @@ -87,19 +87,9 @@ An example event for `am_access` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | -| client.domain | The domain name of the client system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| client.port | Port of the client. | long | | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | -| event.duration | Duration of the event in nanoseconds. If `event.start` and `event.end` are known this value should be the difference between the end and start time. | long | -| event.id | Unique ID to describe the event. | keyword | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | | forgerock.eventName | The name of the audit event. | keyword | | forgerock.http.request.headers.\* | The headers of the HTTP request. | object | | forgerock.http.request.headers.accept | The accept parameter for the request. | keyword | @@ -138,19 +128,7 @@ An example event for `am_access` looks as following: | forgerock.topic | The topic of the event. | keyword | | forgerock.trackingIds | Specifies a unique random string generated as an alias for each AM session ID and OAuth 2.0 token. | keyword | | http.request.Path | The path of the HTTP request. | keyword | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.response.body.content | The full HTTP response body. | wildcard | -| http.response.body.content.text | Multi-field of `http.response.body.content`. | match_only_text | -| http.response.status_code | HTTP response status code. | long | | input.type | Input type | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| server.ip | IP address of the server (IPv4 or IPv6). | ip | -| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| transaction.id | Unique identifier of the transaction within the scope of its trace. A transaction is the highest level of work measured within a service, such as a request to a server. | keyword | -| user.id | Unique identifier of the user. | keyword | ### AM_Activity events @@ -235,12 +213,6 @@ An example event for `am_activity` looks as following: | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | -| event.duration | Duration of the event in nanoseconds. If `event.start` and `event.end` are known this value should be the difference between the end and start time. | long | -| event.id | Unique ID to describe the event. | keyword | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | forgerock.after.\* | Specifies the JSON representation of the object after the activity. | object | | forgerock.before.\* | Specifies the JSON representation of the object prior to the activity. | object | | forgerock.changedFields | Specifies the fields that were changed. | keyword | @@ -252,12 +224,6 @@ An example event for `am_activity` looks as following: | forgerock.topic | The topic of the event. | keyword | | forgerock.trackingIds | Specifies a unique random string generated as an alias for each AM session ID and OAuth 2.0 token. | keyword | | input.type | Input type | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| transaction.id | Unique identifier of the transaction within the scope of its trace. A transaction is the highest level of work measured within a service, such as a request to a server. | keyword | -| user.effective.id | Unique identifier of the user. | keyword | -| user.id | Unique identifier of the user. | keyword | ### AM_Authentication events @@ -356,11 +322,6 @@ An example event for `am_authentication` looks as following: | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | | forgerock.entries | The JSON representation of the details of an authentication module, chain, tree, or node. | flattened | | forgerock.eventName | The name of the audit event. | keyword | | forgerock.level | The log level. | keyword | @@ -370,11 +331,6 @@ An example event for `am_authentication` looks as following: | forgerock.topic | The topic of the event. | keyword | | forgerock.trackingIds | Specifies a unique random string generated as an alias for each AM session ID and OAuth 2.0 token. | keyword | | input.type | Input type | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| transaction.id | Unique identifier of the transaction within the scope of its trace. A transaction is the highest level of work measured within a service, such as a request to a server. | keyword | -| user.id | Unique identifier of the user. | keyword | ### AM_Config events @@ -459,12 +415,6 @@ An example event for `am_config` looks as following: | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | forgerock.changedFields | Specifies the fields that were changed. | keyword | | forgerock.eventName | The name of the audit event. | keyword | | forgerock.level | The log level. | keyword | @@ -475,11 +425,6 @@ An example event for `am_config` looks as following: | forgerock.topic | The topic of the event. | keyword | | forgerock.trackingIds | Specifies a unique random string generated as an alias for each AM session ID and OAuth 2.0 token. | keyword | | input.type | Input type | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| transaction.id | Unique identifier of the transaction within the scope of its trace. A transaction is the highest level of work measured within a service, such as a request to a server. | keyword | -| user.effective.id | Unique identifier of the user. | keyword | -| user.id | Unique identifier of the user. | keyword | ### AM_Core events @@ -550,21 +495,8 @@ An example event for `am_core` looks as following: | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.stack_trace | The stack trace of this error in plain text. | wildcard | -| error.stack_trace.text | Multi-field of `error.stack_trace`. | match_only_text | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | -| event.reason | Reason why this event happened, according to the source. This describes the why of a particular action or outcome captured in the event. Where `event.action` captures the action from the event, `event.reason` describes why that action was taken. For example, a web proxy with an `event.action` which denied the request may also populate `event.reason` with the reason why (e.g. `blocked site`). | keyword | | forgerock.context | The context of the debug event. | keyword | | input.type | Input type | keyword | -| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | -| log.logger | The name of the logger inside an application. This is usually the name of the class which initialized the logger, or can be a custom name. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| transaction.id | Unique identifier of the transaction within the scope of its trace. A transaction is the highest level of work measured within a service, such as a request to a server. | keyword | ### IDM_access events @@ -677,14 +609,9 @@ An example event for `idm_access` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| client.port | Port of the client. | long | | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | forgerock.eventName | The name of the audit event. | keyword | | forgerock.http.request.headers.host | The host header of the HTTP request. | keyword | | forgerock.http.request.secure | A flag describing whether or not the HTTP request was secure. | boolean | @@ -698,14 +625,7 @@ An example event for `idm_access` looks as following: | forgerock.source | The source of the event. | keyword | | forgerock.topic | The topic of the event. | keyword | | http.request.Path | The path of the HTTP request. | keyword | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.response.status_code | HTTP response status code. | long | | input.type | Input type | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| server.ip | IP address of the server (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| transaction.id | Unique identifier of the transaction within the scope of its trace. A transaction is the highest level of work measured within a service, such as a request to a server. | keyword | -| user.id | Unique identifier of the user. | keyword | ### IDM_activity events @@ -787,10 +707,6 @@ An example event for `idm_activity` looks as following: | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | forgerock.eventName | The name of the audit event. | keyword | | forgerock.level | The log level. | keyword | | forgerock.message | Human readable text about the action. | keyword | @@ -801,11 +717,6 @@ An example event for `idm_activity` looks as following: | forgerock.source | The source of the event. | keyword | | forgerock.topic | The topic of the event. | keyword | | input.type | Input type | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| transaction.id | Unique identifier of the transaction within the scope of its trace. A transaction is the highest level of work measured within a service, such as a request to a server. | keyword | -| user.effective.id | Unique identifier of the user. | keyword | -| user.id | Unique identifier of the user. | keyword | ### IDM_authentication events @@ -900,10 +811,6 @@ An example event for `idm_authentication` looks as following: | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | forgerock.entries | The JSON representation of the details of an authentication module, chain, tree, or node. | flattened | | forgerock.eventName | The name of the audit event. | keyword | | forgerock.level | The log level. | keyword | @@ -913,10 +820,6 @@ An example event for `idm_authentication` looks as following: | forgerock.topic | The topic of the event. | keyword | | forgerock.trackingIds | Specifies a unique random string generated as an alias for each AM session ID and OAuth 2.0 token. | keyword | | input.type | Input type | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| transaction.id | Unique identifier of the transaction within the scope of its trace. A transaction is the highest level of work measured within a service, such as a request to a server. | keyword | -| user.id | Unique identifier of the user. | keyword | ### IDM_config events @@ -999,10 +902,6 @@ An example event for `idm_config` looks as following: | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | forgerock.changedFields | Specifies the fields that were changed. | keyword | | forgerock.eventName | The name of the audit event. | keyword | | forgerock.level | The log level. | keyword | @@ -1010,11 +909,6 @@ An example event for `idm_config` looks as following: | forgerock.source | The source of the event. | keyword | | forgerock.topic | The topic of the event. | keyword | | input.type | Input type | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| transaction.id | Unique identifier of the transaction within the scope of its trace. A transaction is the highest level of work measured within a service, such as a request to a server. | keyword | -| user.effective.id | Unique identifier of the user. | keyword | -| user.id | Unique identifier of the user. | keyword | ### IDM_core events @@ -1075,12 +969,7 @@ An example event for `idm_core` looks as following: | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | input.type | Input type | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| tags | List of keywords used to tag each event. | keyword | ### IDM_sync events @@ -1159,11 +1048,6 @@ An example event for `idm_sync` looks as following: | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | | forgerock.action | The synchronization action, depicted as a Common REST action. | keyword | | forgerock.eventName | The name of the audit event. | keyword | | forgerock.level | The log level. | keyword | @@ -1175,7 +1059,3 @@ An example event for `idm_sync` looks as following: | forgerock.targetObjectId | Object ID on the target system | keyword | | forgerock.topic | The topic of the event. | keyword | | input.type | Input type | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| transaction.id | Unique identifier of the transaction within the scope of its trace. A transaction is the highest level of work measured within a service, such as a request to a server. | keyword | -| user.id | Unique identifier of the user. | keyword | diff --git a/packages/forgerock/manifest.yml b/packages/forgerock/manifest.yml index b5e583b2db4..a551fd0f365 100644 --- a/packages/forgerock/manifest.yml +++ b/packages/forgerock/manifest.yml @@ -1,13 +1,13 @@ name: forgerock title: "ForgeRock" -version: "1.17.1" +version: "1.18.0" description: Collect audit logs from ForgeRock with Elastic Agent. type: integration format_version: "3.0.2" categories: ["security"] conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" screenshots: - src: /img/forgerock-dashboard.png title: ForgeRock Dashboard From e48deab10ab950c91e052e2015d7912a81966a51 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:24:44 +0930 Subject: [PATCH 043/121] [gcp_pubsub] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. The ecs.version in sample_event.json files was changed to 8.11.0. Previously sample_event.json files contained 8.0.0. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/gcp_pubsub --- packages/gcp_pubsub/changelog.yml | 5 + packages/gcp_pubsub/fields/agent.yml | 167 +------------------------- packages/gcp_pubsub/fields/ecs.yml | 10 -- packages/gcp_pubsub/manifest.yml | 5 +- packages/gcp_pubsub/sample_event.json | 2 +- 5 files changed, 9 insertions(+), 180 deletions(-) delete mode 100644 packages/gcp_pubsub/fields/ecs.yml diff --git a/packages/gcp_pubsub/changelog.yml b/packages/gcp_pubsub/changelog.yml index 85e3d5a29c9..17b81567ff2 100644 --- a/packages/gcp_pubsub/changelog.yml +++ b/packages/gcp_pubsub/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.1.0" + changes: + - description: ECS version updated to 8.11.0. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.0.0" changes: - description: Converted Google Pub/Sub to input package type. diff --git a/packages/gcp_pubsub/fields/agent.yml b/packages/gcp_pubsub/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/gcp_pubsub/fields/agent.yml +++ b/packages/gcp_pubsub/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/gcp_pubsub/fields/ecs.yml b/packages/gcp_pubsub/fields/ecs.yml deleted file mode 100644 index c565c4a26fe..00000000000 --- a/packages/gcp_pubsub/fields/ecs.yml +++ /dev/null @@ -1,10 +0,0 @@ -- name: ecs.version - external: ecs -- name: log.level - external: ecs -- name: message - external: ecs -- name: event.original - external: ecs -- name: tags - external: ecs diff --git a/packages/gcp_pubsub/manifest.yml b/packages/gcp_pubsub/manifest.yml index bd457097361..a16e8457b73 100644 --- a/packages/gcp_pubsub/manifest.yml +++ b/packages/gcp_pubsub/manifest.yml @@ -3,7 +3,7 @@ title: Custom Google Pub/Sub Logs format_version: "3.0.2" description: Collect Logs from Google Pub/Sub topics type: input -version: "2.0.0" +version: "2.1.0" icons: - src: /img/logo_gcp.svg title: logo gcp @@ -15,7 +15,7 @@ categories: - custom conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" policy_templates: - name: gcp title: Custom Google Pub/Sub Logs @@ -109,7 +109,6 @@ policy_templates: show_user: false description: > Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. - - name: alternative_host type: text title: Alternative host diff --git a/packages/gcp_pubsub/sample_event.json b/packages/gcp_pubsub/sample_event.json index 7360eda8ff4..671bd5cf771 100644 --- a/packages/gcp_pubsub/sample_event.json +++ b/packages/gcp_pubsub/sample_event.json @@ -13,7 +13,7 @@ "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "94455c8e-3b6c-40c3-96b7-9163c5f086a0", From 75a8145d2943289770d3d52de52d3ad77db6ce34 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:24:49 +0930 Subject: [PATCH 044/121] [github] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/github --- packages/github/changelog.yml | 5 + .../github/data_stream/audit/fields/agent.yml | 93 +---------- .../github/data_stream/audit/fields/ecs.yml | 54 ------- .../code_scanning/fields/agent.yml | 93 +---------- .../data_stream/code_scanning/fields/ecs.yml | 20 --- .../data_stream/dependabot/fields/agent.yml | 93 +---------- .../data_stream/dependabot/fields/ecs.yml | 36 ----- .../data_stream/issues/fields/agent.yml | 93 +---------- .../github/data_stream/issues/fields/ecs.yml | 22 --- .../secret_scanning/fields/agent.yml | 93 +---------- .../secret_scanning/fields/ecs.yml | 10 -- packages/github/docs/README.md | 148 ------------------ packages/github/manifest.yml | 4 +- 13 files changed, 12 insertions(+), 752 deletions(-) delete mode 100644 packages/github/data_stream/audit/fields/ecs.yml delete mode 100644 packages/github/data_stream/code_scanning/fields/ecs.yml delete mode 100644 packages/github/data_stream/dependabot/fields/ecs.yml delete mode 100644 packages/github/data_stream/issues/fields/ecs.yml delete mode 100644 packages/github/data_stream/secret_scanning/fields/ecs.yml diff --git a/packages/github/changelog.yml b/packages/github/changelog.yml index cedc2bfec30..473548be21e 100644 --- a/packages/github/changelog.yml +++ b/packages/github/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.29.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.28.0" changes: - description: Set sensitive values as secret and fix incorrect mappings. diff --git a/packages/github/data_stream/audit/fields/agent.yml b/packages/github/data_stream/audit/fields/agent.yml index 4d9a6f7b362..bc42d0a853b 100644 --- a/packages/github/data_stream/audit/fields/agent.yml +++ b/packages/github/data_stream/audit/fields/agent.yml @@ -1,100 +1,9 @@ - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/github/data_stream/audit/fields/ecs.yml b/packages/github/data_stream/audit/fields/ecs.yml deleted file mode 100644 index 4c860994252..00000000000 --- a/packages/github/data_stream/audit/fields/ecs.yml +++ /dev/null @@ -1,54 +0,0 @@ -- name: client.geo.country_iso_code - external: ecs -- name: ecs.version - external: ecs -- name: error.message - external: ecs -- name: event.action - external: ecs -- name: event.category - external: ecs -- name: event.created - external: ecs -- name: event.id - external: ecs -- name: event.ingested - external: ecs -- name: event.kind - external: ecs -- name: event.original - external: ecs -- name: event.outcome - external: ecs -- name: event.type - external: ecs -- name: message - external: ecs -- name: related.user - external: ecs -- name: user.name - external: ecs -- name: group.name - external: ecs -- name: related.ip - external: ecs -- name: user.target.group.name - external: ecs -- name: user.target.name - external: ecs -- name: user_agent.device.name - external: ecs -- name: user_agent.name - external: ecs -- name: user_agent.original - external: ecs -- name: user_agent.os.full - external: ecs -- name: user_agent.os.name - external: ecs -- name: user_agent.os.version - external: ecs -- name: user_agent.version - external: ecs -- name: tags - external: ecs diff --git a/packages/github/data_stream/code_scanning/fields/agent.yml b/packages/github/data_stream/code_scanning/fields/agent.yml index 4d9a6f7b362..bc42d0a853b 100644 --- a/packages/github/data_stream/code_scanning/fields/agent.yml +++ b/packages/github/data_stream/code_scanning/fields/agent.yml @@ -1,100 +1,9 @@ - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/github/data_stream/code_scanning/fields/ecs.yml b/packages/github/data_stream/code_scanning/fields/ecs.yml deleted file mode 100644 index b900cad0bca..00000000000 --- a/packages/github/data_stream/code_scanning/fields/ecs.yml +++ /dev/null @@ -1,20 +0,0 @@ -- name: ecs.version - external: ecs -- name: error.message - external: ecs -- name: event.category - external: ecs -- name: event.created - external: ecs -- name: event.kind - external: ecs -- name: rule.id - external: ecs -- name: rule.description - external: ecs -- name: rule.name - external: ecs -- name: tags - external: ecs -- name: message - external: ecs diff --git a/packages/github/data_stream/dependabot/fields/agent.yml b/packages/github/data_stream/dependabot/fields/agent.yml index 4d9a6f7b362..bc42d0a853b 100644 --- a/packages/github/data_stream/dependabot/fields/agent.yml +++ b/packages/github/data_stream/dependabot/fields/agent.yml @@ -1,100 +1,9 @@ - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/github/data_stream/dependabot/fields/ecs.yml b/packages/github/data_stream/dependabot/fields/ecs.yml deleted file mode 100644 index 627371d1ed3..00000000000 --- a/packages/github/data_stream/dependabot/fields/ecs.yml +++ /dev/null @@ -1,36 +0,0 @@ -- name: ecs.version - external: ecs -- name: error.message - external: ecs -- name: event.category - external: ecs -- name: event.created - external: ecs -- name: event.start - external: ecs -- name: event.end - external: ecs -- name: event.duration - external: ecs -- name: event.kind - external: ecs -- name: vulnerability.classification - external: ecs -- name: vulnerability.description - external: ecs -- name: vulnerability.enumeration - external: ecs -- name: vulnerability.id - external: ecs -- name: vulnerability.reference - external: ecs -- name: vulnerability.scanner.vendor - external: ecs -- name: vulnerability.score.base - external: ecs -- name: vulnerability.score.version - external: ecs -- name: vulnerability.severity - external: ecs -- name: tags - external: ecs diff --git a/packages/github/data_stream/issues/fields/agent.yml b/packages/github/data_stream/issues/fields/agent.yml index 4d9a6f7b362..bc42d0a853b 100644 --- a/packages/github/data_stream/issues/fields/agent.yml +++ b/packages/github/data_stream/issues/fields/agent.yml @@ -1,100 +1,9 @@ - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/github/data_stream/issues/fields/ecs.yml b/packages/github/data_stream/issues/fields/ecs.yml deleted file mode 100644 index f9f05af6577..00000000000 --- a/packages/github/data_stream/issues/fields/ecs.yml +++ /dev/null @@ -1,22 +0,0 @@ -- name: ecs.version - external: ecs -- name: error.message - external: ecs -- name: event.category - external: ecs -- name: event.created - external: ecs -- name: event.kind - external: ecs -- name: user.name - external: ecs -- name: user.id - external: ecs -- name: user.roles - external: ecs -- name: related.user - external: ecs -- name: tags - external: ecs -- name: message - external: ecs diff --git a/packages/github/data_stream/secret_scanning/fields/agent.yml b/packages/github/data_stream/secret_scanning/fields/agent.yml index 4d9a6f7b362..bc42d0a853b 100644 --- a/packages/github/data_stream/secret_scanning/fields/agent.yml +++ b/packages/github/data_stream/secret_scanning/fields/agent.yml @@ -1,100 +1,9 @@ - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/github/data_stream/secret_scanning/fields/ecs.yml b/packages/github/data_stream/secret_scanning/fields/ecs.yml deleted file mode 100644 index f25bf7ec020..00000000000 --- a/packages/github/data_stream/secret_scanning/fields/ecs.yml +++ /dev/null @@ -1,10 +0,0 @@ -- name: ecs.version - external: ecs -- name: error.message - external: ecs -- name: event.category - external: ecs -- name: event.created - external: ecs -- name: tags - external: ecs diff --git a/packages/github/docs/README.md b/packages/github/docs/README.md index 0ca08ce09bb..99c357a34f8 100644 --- a/packages/github/docs/README.md +++ b/packages/github/docs/README.md @@ -20,23 +20,11 @@ To use this integration, the following prerequisites must be met: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| client.geo.country_iso_code | Country ISO code. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | github.actor_ip | The IP address of the entity performing the action. | ip | | github.category | GitHub action category. | keyword | | github.hashed_token | SHA-256 hash of the token used for authentication. | keyword | @@ -51,44 +39,10 @@ To use this integration, the following prerequisites must be met: | github.repository_selection | Whether all repositories have been selected or there's a selection involved. | keyword | | github.team | GitHub team name. | keyword | | github.user_agent | The user agent of the entity performing the action. | keyword | -| group.name | Name of the group. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user.target.group.name | Name of the group. | keyword | -| user.target.name | Short name or login of the user. | keyword | -| user.target.name.text | Multi-field of `user.target.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | An example event for `audit` looks as following: @@ -173,12 +127,7 @@ Or use a personal access token with the `security_events` scope for private repo | data_stream.dataset | Data stream dataset name. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | -| error.message | Error message. | match_only_text | | | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | | | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | | | event.dataset | Event dataset | constant_keyword | | | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | | | event.module | Event module | constant_keyword | | | | github.code_scanning.created_at | The time that the alert was created in ISO 8601 format: `YYYY-MM-DDTHH:MM:SSZ` | date | | | | github.code_scanning.dismissed_at | The time that the alert was dismissed in ISO 8601 format: `YYYY-MM-DDTHH:MM:SSZ`. | date | | | @@ -236,29 +185,10 @@ Or use a personal access token with the `security_events` scope for private repo | github.repository.url | The URL to get more information about the repository from the GitHub API. | keyword | | | | github.severity | The security severity of the alert | keyword | | | | github.state | State of a code scanning alert | keyword | | | -| host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | | -| host.ip | Host ip addresses. | ip | | | -| host.mac | Host mac addresses. | keyword | | | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | | | host.os.build | OS build information. | keyword | | | | host.os.codename | OS codename, if any. | keyword | | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | | -| host.os.name | Operating system name, without the version. | keyword | | | -| host.os.name.text | Multi-field of `host.os.name`. | text | | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | | -| host.os.version | Operating system version as a raw string. | keyword | | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | | | input.type | Type of Filebeat input. | keyword | | | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | | -| rule.description | The description of the rule generating the event. | keyword | | | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | | | -| rule.name | The name of the rule or signature generating the event. | keyword | | | -| tags | List of keywords used to tag each event. | keyword | | | An example event for `code_scanning` looks as following: @@ -372,10 +302,6 @@ Or you must be an administrator for the repository or for the organization that | data_stream.dataset | Data stream dataset name. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | -| error.message | Error message. | match_only_text | | | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | | | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | | | event.dataset | Event dataset | constant_keyword | | | | event.module | Event module | constant_keyword | | | | github.repository.html_url | The URL to view the repository on GitHub.com. | keyword | | | @@ -417,25 +343,10 @@ Or you must be an administrator for the repository or for the organization that | github.secret_scanning.url | The REST API URL of the alert resource | keyword | | | | github.severity | The severity of the secret scanning alert | keyword | | | | github.state | State of a code scanning alert | keyword | | | -| host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | | -| host.ip | Host ip addresses. | ip | | | -| host.mac | Host mac addresses. | keyword | | | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | | | host.os.build | OS build information. | keyword | | | | host.os.codename | OS codename, if any. | keyword | | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | | -| host.os.name | Operating system name, without the version. | keyword | | | -| host.os.name.text | Multi-field of `host.os.name`. | text | | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | | -| host.os.version | Operating system version as a raw string. | keyword | | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | | | input.type | Type of Filebeat input. | keyword | | | -| tags | List of keywords used to tag each event. | keyword | | | An example event for `secret_scanning` looks as following: @@ -532,16 +443,8 @@ To use this integration, you must be an administrator for the repository or for | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.duration | Duration of the event in nanoseconds. If `event.start` and `event.end` are known this value should be the difference between the end and start time. | long | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | | github.dependabot.created_at | When was the alert created | date | | github.dependabot.dependabot_update.error.body | The body of the error. | text | | github.dependabot.dependabot_update.error.error_type | The error code. | keyword | @@ -594,35 +497,10 @@ To use this integration, you must be an administrator for the repository or for | github.repository.url | The HTTP URL for this repository. | keyword | | github.severity | The severity of the advisory. | keyword | | github.state | Identifies the state of the alert. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| vulnerability.classification | The classification of the vulnerability scoring system. For example (https://www.first.org/cvss/) | keyword | -| vulnerability.description | The description of the vulnerability that provides additional context of the vulnerability. For example (https://cve.mitre.org/about/faqs.html#cve_entry_descriptions_created[Common Vulnerabilities and Exposure CVE description]) | keyword | -| vulnerability.description.text | Multi-field of `vulnerability.description`. | match_only_text | -| vulnerability.enumeration | The type of identifier used for this vulnerability. For example (https://cve.mitre.org/about/) | keyword | -| vulnerability.id | The identification (ID) is the number portion of a vulnerability entry. It includes a unique identification number for the vulnerability. For example (https://cve.mitre.org/about/faqs.html#what_is_cve_id)[Common Vulnerabilities and Exposure CVE ID] | keyword | -| vulnerability.reference | A resource that provides additional information, context, and mitigations for the identified vulnerability. | keyword | -| vulnerability.scanner.vendor | The name of the vulnerability scanner vendor. | keyword | -| vulnerability.score.base | Scores can range from 0.0 to 10.0, with 10.0 being the most severe. Base scores cover an assessment for exploitability metrics (attack vector, complexity, privileges, and user interaction), impact metrics (confidentiality, integrity, and availability), and scope. For example (https://www.first.org/cvss/specification-document) | float | -| vulnerability.score.version | The National Vulnerability Database (NVD) provides qualitative severity rankings of "Low", "Medium", and "High" for CVSS v2.0 base score ranges in addition to the severity ratings for CVSS v3.0 as they are defined in the CVSS v3.0 specification. CVSS is owned and managed by FIRST.Org, Inc. (FIRST), a US-based non-profit organization, whose mission is to help computer security incident response teams across the world. For example (https://nvd.nist.gov/vuln-metrics/cvss) | keyword | -| vulnerability.severity | The severity of the vulnerability can help with metrics and internal prioritization regarding remediation. For example (https://nvd.nist.gov/vuln-metrics/cvss) | keyword | An example event for `dependabot` looks as following: @@ -778,12 +656,7 @@ To use this integration, users must use Github Apps or Personal Access Token wit | data_stream.dataset | Data stream dataset name. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | -| error.message | Error message. | match_only_text | | | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | | | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | | | event.dataset | Event dataset | constant_keyword | | | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | | | event.module | Event module | constant_keyword | | | | github.issues.active_lock_reason | | keyword | | | | github.issues.assignee.email | | keyword | | | @@ -848,31 +721,10 @@ To use this integration, users must use Github Apps or Personal Access Token wit | github.repository.owner.login | | keyword | | | | github.repository.url | The URL to get more information about the repository from the GitHub API. | keyword | | | | github.state | State of github issue | keyword | | | -| host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | | -| host.ip | Host ip addresses. | ip | | | -| host.mac | Host mac addresses. | keyword | | | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | | | host.os.build | OS build information. | keyword | | | | host.os.codename | OS codename, if any. | keyword | | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | | -| host.os.name | Operating system name, without the version. | keyword | | | -| host.os.name.text | Multi-field of `host.os.name`. | text | | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | | -| host.os.version | Operating system version as a raw string. | keyword | | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | | | input.type | Type of Filebeat input. | keyword | | | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | | -| related.user | All the user names or other user identifiers seen on the event. | keyword | | | -| tags | List of keywords used to tag each event. | keyword | | | -| user.id | Unique identifier of the user. | keyword | | | -| user.name | Short name or login of the user. | keyword | | | -| user.name.text | Multi-field of `user.name`. | match_only_text | | | -| user.roles | Array of user roles at the time of the event. | keyword | | | An example event for `issues` looks as following: diff --git a/packages/github/manifest.yml b/packages/github/manifest.yml index e3b8ab54974..e0ae59529b7 100644 --- a/packages/github/manifest.yml +++ b/packages/github/manifest.yml @@ -1,13 +1,13 @@ name: github title: GitHub -version: "1.28.0" +version: "1.29.0" description: Collect logs from GitHub with Elastic Agent. type: integration format_version: "3.0.2" categories: [security, "productivity_security"] conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" icons: - src: /img/github.svg title: GitHub From ad230d87bd6ccc3077970b0ec46d2fa5c96cb720 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:24:51 +0930 Subject: [PATCH 045/121] [gitlab] - Updated fields definitions Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/gitlab --- packages/gitlab/changelog.yml | 5 +++++ packages/gitlab/data_stream/api/fields/agent.yml | 11 ----------- .../gitlab/data_stream/production/fields/agent.yml | 11 ----------- packages/gitlab/docs/README.md | 2 -- packages/gitlab/manifest.yml | 2 +- 5 files changed, 6 insertions(+), 25 deletions(-) diff --git a/packages/gitlab/changelog.yml b/packages/gitlab/changelog.yml index 42e79b8cbd7..dee57e4dbf5 100644 --- a/packages/gitlab/changelog.yml +++ b/packages/gitlab/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.2.0" + changes: + - description: Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: 0.1.0 changes: - description: Initial Version diff --git a/packages/gitlab/data_stream/api/fields/agent.yml b/packages/gitlab/data_stream/api/fields/agent.yml index 27f215b1cd6..df92bfa51a9 100644 --- a/packages/gitlab/data_stream/api/fields/agent.yml +++ b/packages/gitlab/data_stream/api/fields/agent.yml @@ -8,17 +8,6 @@ - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - name: host title: Host group: 2 diff --git a/packages/gitlab/data_stream/production/fields/agent.yml b/packages/gitlab/data_stream/production/fields/agent.yml index 27f215b1cd6..df92bfa51a9 100644 --- a/packages/gitlab/data_stream/production/fields/agent.yml +++ b/packages/gitlab/data_stream/production/fields/agent.yml @@ -8,17 +8,6 @@ - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - name: host title: Host group: 2 diff --git a/packages/gitlab/docs/README.md b/packages/gitlab/docs/README.md index 19016f94ca4..e5dd1b129bf 100644 --- a/packages/gitlab/docs/README.md +++ b/packages/gitlab/docs/README.md @@ -28,7 +28,6 @@ Collect logs for HTTP requests made to the GitLab API. Check out the [GitLab API |---|---|---| | @timestamp | Event timestamp. | date | | cloud.image.id | Image ID for the cloud instance. | keyword | -| container.labels | Image labels. | object | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | @@ -297,7 +296,6 @@ Collect logs for Rails controller requests received from GitLab. Check out the [ |---|---|---| | @timestamp | Event timestamp. | date | | cloud.image.id | Image ID for the cloud instance. | keyword | -| container.labels | Image labels. | object | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | diff --git a/packages/gitlab/manifest.yml b/packages/gitlab/manifest.yml index 2a5d8c3dda5..7010196d1b2 100644 --- a/packages/gitlab/manifest.yml +++ b/packages/gitlab/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.1.3 name: gitlab title: GitLab -version: 0.1.0 +version: "0.2.0" description: Collect logs from GitLab with Elastic Agent. type: integration categories: From f44a10c84c0028e34ebe1336989a1a14b01d9969 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:24:51 +0930 Subject: [PATCH 046/121] [google_cloud_storage] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. The ecs.version in sample_event.json files was changed to 8.11.0. Previously sample_event.json files contained 8.0.0. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/google_cloud_storage --- packages/google_cloud_storage/_dev/build/build.yml | 1 - packages/google_cloud_storage/changelog.yml | 5 +++++ packages/google_cloud_storage/fields/agent.yml | 3 --- packages/google_cloud_storage/fields/beats.yml | 3 --- packages/google_cloud_storage/manifest.yml | 6 ++---- packages/google_cloud_storage/sample_event.json | 2 +- 6 files changed, 8 insertions(+), 12 deletions(-) diff --git a/packages/google_cloud_storage/_dev/build/build.yml b/packages/google_cloud_storage/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/google_cloud_storage/_dev/build/build.yml +++ b/packages/google_cloud_storage/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/google_cloud_storage/changelog.yml b/packages/google_cloud_storage/changelog.yml index c4ab4a18052..70cea1a2a95 100644 --- a/packages/google_cloud_storage/changelog.yml +++ b/packages/google_cloud_storage/changelog.yml @@ -1,3 +1,8 @@ +- version: "2.1.0" + changes: + - description: ECS version updated to 8.11.0. Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.0.0" changes: - description: Convert Google Cloud Storage to input package type. diff --git a/packages/google_cloud_storage/fields/agent.yml b/packages/google_cloud_storage/fields/agent.yml index 230f7bc911d..9638d2992eb 100644 --- a/packages/google_cloud_storage/fields/agent.yml +++ b/packages/google_cloud_storage/fields/agent.yml @@ -5,9 +5,6 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. diff --git a/packages/google_cloud_storage/fields/beats.yml b/packages/google_cloud_storage/fields/beats.yml index 8c03b061f7c..6d9a7862671 100644 --- a/packages/google_cloud_storage/fields/beats.yml +++ b/packages/google_cloud_storage/fields/beats.yml @@ -1,9 +1,6 @@ - name: input.type description: Type of Filebeat input. type: keyword -- name: tags - type: keyword - description: User defined tags - name: log.offset type: long description: Log offset diff --git a/packages/google_cloud_storage/manifest.yml b/packages/google_cloud_storage/manifest.yml index 6cd68fa3d28..ea3b4d4ae83 100644 --- a/packages/google_cloud_storage/manifest.yml +++ b/packages/google_cloud_storage/manifest.yml @@ -3,10 +3,10 @@ name: google_cloud_storage title: Custom GCS (Google Cloud Storage) Input description: Collect JSON data from configured GCS Bucket with Elastic Agent. type: input -version: 2.0.0 +version: "2.1.0" conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" categories: - custom - cloud @@ -105,7 +105,6 @@ policy_templates: title: Buckets description: > This attribute contains the details about a specific bucket like, name, number_of_workers, poll, poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, while the fields number_of_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information. - required: true show_user: true default: | @@ -131,7 +130,6 @@ policy_templates: # - regex: "event/" description: > If the GCS bucket will have events that correspond to files that this integration shouldn’t process, file_selectors can be used to limit the files that are downloaded. This is a list of selectors which is made up of regex patters. The regex should match the GCS bucket filepath. Regexes use [RE2 syntax](https://pkg.go.dev/regexp/syntax). Files that don’t match one of the regexes will not be processed. - - name: timestamp_epoch type: integer title: Timestamp Epoch diff --git a/packages/google_cloud_storage/sample_event.json b/packages/google_cloud_storage/sample_event.json index fec33721806..37ef5bb15b6 100644 --- a/packages/google_cloud_storage/sample_event.json +++ b/packages/google_cloud_storage/sample_event.json @@ -13,7 +13,7 @@ "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "80442ebe-168f-468b-82af-30451478d848", From 54050337253e359858b0857b1bf8b1692844e7a5 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:24:54 +0930 Subject: [PATCH 047/121] [google_scc] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/google_scc --- packages/google_scc/_dev/build/build.yml | 1 - packages/google_scc/changelog.yml | 5 +++++ packages/google_scc/data_stream/asset/fields/beats.yml | 3 --- packages/google_scc/data_stream/audit/fields/beats.yml | 3 --- packages/google_scc/data_stream/finding/fields/beats.yml | 3 --- packages/google_scc/data_stream/source/fields/beats.yml | 3 --- packages/google_scc/docs/README.md | 4 ---- packages/google_scc/manifest.yml | 4 ++-- 8 files changed, 7 insertions(+), 19 deletions(-) diff --git a/packages/google_scc/_dev/build/build.yml b/packages/google_scc/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/google_scc/_dev/build/build.yml +++ b/packages/google_scc/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/google_scc/changelog.yml b/packages/google_scc/changelog.yml index 7efebc089a1..60b60614bf2 100644 --- a/packages/google_scc/changelog.yml +++ b/packages/google_scc/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.4.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.3.0" changes: - description: Improve handling of empty responses. diff --git a/packages/google_scc/data_stream/asset/fields/beats.yml b/packages/google_scc/data_stream/asset/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/google_scc/data_stream/asset/fields/beats.yml +++ b/packages/google_scc/data_stream/asset/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/google_scc/data_stream/audit/fields/beats.yml b/packages/google_scc/data_stream/audit/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/google_scc/data_stream/audit/fields/beats.yml +++ b/packages/google_scc/data_stream/audit/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/google_scc/data_stream/finding/fields/beats.yml b/packages/google_scc/data_stream/finding/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/google_scc/data_stream/finding/fields/beats.yml +++ b/packages/google_scc/data_stream/finding/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/google_scc/data_stream/source/fields/beats.yml b/packages/google_scc/data_stream/source/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/google_scc/data_stream/source/fields/beats.yml +++ b/packages/google_scc/data_stream/source/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/google_scc/docs/README.md b/packages/google_scc/docs/README.md index 44d9b3b8c08..16af6a4f8bd 100644 --- a/packages/google_scc/docs/README.md +++ b/packages/google_scc/docs/README.md @@ -495,7 +495,6 @@ An example event for `asset` looks as following: | google_scc.asset.window.start_time | | date | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Finding @@ -782,7 +781,6 @@ An example event for `finding` looks as following: | google_scc.finding.vulnerability.cve.upstream_fix_available | Whether upstream fix is available for the CVE. | boolean | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Source @@ -867,7 +865,6 @@ An example event for `source` looks as following: | google_scc.source.name | The relative resource name of this source. See: https://cloud.google.com/apis/design/resource_names#relative_resource_name Example: "organizations/\{organization_id\}/sources/\{source_id\}". | keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Audit @@ -1086,4 +1083,3 @@ An example event for `audit` looks as following: | google_scc.audit.trace_sampled | The sampling decision of the trace associated with the log entry. | boolean | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | diff --git a/packages/google_scc/manifest.yml b/packages/google_scc/manifest.yml index 43f88d620e7..2f4783c5ffd 100644 --- a/packages/google_scc/manifest.yml +++ b/packages/google_scc/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: google_scc title: Google Security Command Center -version: "1.3.0" +version: "1.4.0" description: Collect logs from Google Security Command Center with Elastic Agent. type: integration categories: @@ -10,7 +10,7 @@ categories: - cloudsecurity_cdr conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: subscription: basic screenshots: From 668ebb457bf8a02bbf86a5fdfe20b5525895616f Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:25:12 +0930 Subject: [PATCH 048/121] [google_workspace] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/google_workspace --- packages/google_workspace/_dev/build/build.yml | 1 - packages/google_workspace/changelog.yml | 5 +++++ .../access_transparency/fields/beats.yml | 3 --- .../data_stream/admin/fields/beats.yml | 3 --- .../data_stream/alert/fields/beats.yml | 3 --- .../context_aware_access/fields/beats.yml | 3 --- .../data_stream/device/fields/beats.yml | 3 --- .../data_stream/drive/fields/beats.yml | 3 --- .../data_stream/gcp/fields/beats.yml | 3 --- .../data_stream/group_enterprise/fields/beats.yml | 3 --- .../data_stream/groups/fields/beats.yml | 3 --- .../data_stream/login/fields/beats.yml | 3 --- .../data_stream/rules/fields/beats.yml | 3 --- .../data_stream/saml/fields/beats.yml | 3 --- .../data_stream/token/fields/beats.yml | 3 --- .../data_stream/user_accounts/fields/beats.yml | 3 --- packages/google_workspace/docs/README.md | 14 -------------- packages/google_workspace/manifest.yml | 4 ++-- 18 files changed, 7 insertions(+), 59 deletions(-) diff --git a/packages/google_workspace/_dev/build/build.yml b/packages/google_workspace/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/google_workspace/_dev/build/build.yml +++ b/packages/google_workspace/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/google_workspace/changelog.yml b/packages/google_workspace/changelog.yml index ace3e23ac05..5ad201bf890 100644 --- a/packages/google_workspace/changelog.yml +++ b/packages/google_workspace/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.23.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.22.0" changes: - description: Improve handling of empty responses. diff --git a/packages/google_workspace/data_stream/access_transparency/fields/beats.yml b/packages/google_workspace/data_stream/access_transparency/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/google_workspace/data_stream/access_transparency/fields/beats.yml +++ b/packages/google_workspace/data_stream/access_transparency/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/google_workspace/data_stream/admin/fields/beats.yml b/packages/google_workspace/data_stream/admin/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/google_workspace/data_stream/admin/fields/beats.yml +++ b/packages/google_workspace/data_stream/admin/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/google_workspace/data_stream/alert/fields/beats.yml b/packages/google_workspace/data_stream/alert/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/google_workspace/data_stream/alert/fields/beats.yml +++ b/packages/google_workspace/data_stream/alert/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/google_workspace/data_stream/context_aware_access/fields/beats.yml b/packages/google_workspace/data_stream/context_aware_access/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/google_workspace/data_stream/context_aware_access/fields/beats.yml +++ b/packages/google_workspace/data_stream/context_aware_access/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/google_workspace/data_stream/device/fields/beats.yml b/packages/google_workspace/data_stream/device/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/google_workspace/data_stream/device/fields/beats.yml +++ b/packages/google_workspace/data_stream/device/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/google_workspace/data_stream/drive/fields/beats.yml b/packages/google_workspace/data_stream/drive/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/google_workspace/data_stream/drive/fields/beats.yml +++ b/packages/google_workspace/data_stream/drive/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/google_workspace/data_stream/gcp/fields/beats.yml b/packages/google_workspace/data_stream/gcp/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/google_workspace/data_stream/gcp/fields/beats.yml +++ b/packages/google_workspace/data_stream/gcp/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/google_workspace/data_stream/group_enterprise/fields/beats.yml b/packages/google_workspace/data_stream/group_enterprise/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/google_workspace/data_stream/group_enterprise/fields/beats.yml +++ b/packages/google_workspace/data_stream/group_enterprise/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/google_workspace/data_stream/groups/fields/beats.yml b/packages/google_workspace/data_stream/groups/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/google_workspace/data_stream/groups/fields/beats.yml +++ b/packages/google_workspace/data_stream/groups/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/google_workspace/data_stream/login/fields/beats.yml b/packages/google_workspace/data_stream/login/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/google_workspace/data_stream/login/fields/beats.yml +++ b/packages/google_workspace/data_stream/login/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/google_workspace/data_stream/rules/fields/beats.yml b/packages/google_workspace/data_stream/rules/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/google_workspace/data_stream/rules/fields/beats.yml +++ b/packages/google_workspace/data_stream/rules/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/google_workspace/data_stream/saml/fields/beats.yml b/packages/google_workspace/data_stream/saml/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/google_workspace/data_stream/saml/fields/beats.yml +++ b/packages/google_workspace/data_stream/saml/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/google_workspace/data_stream/token/fields/beats.yml b/packages/google_workspace/data_stream/token/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/google_workspace/data_stream/token/fields/beats.yml +++ b/packages/google_workspace/data_stream/token/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/google_workspace/data_stream/user_accounts/fields/beats.yml b/packages/google_workspace/data_stream/user_accounts/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/google_workspace/data_stream/user_accounts/fields/beats.yml +++ b/packages/google_workspace/data_stream/user_accounts/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/google_workspace/docs/README.md b/packages/google_workspace/docs/README.md index 11d81229b57..8bcbbd23640 100644 --- a/packages/google_workspace/docs/README.md +++ b/packages/google_workspace/docs/README.md @@ -275,7 +275,6 @@ An example event for `saml` looks as following: | google_workspace.saml.status_code | SAML status code. | keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### User Accounts @@ -399,7 +398,6 @@ An example event for `user_accounts` looks as following: | google_workspace.user_accounts.email_forwarding_destination_address | Out of domain email the actor has forwarded to. | keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Login Accounts @@ -538,7 +536,6 @@ An example event for `login` looks as following: | google_workspace.organization.domain | The domain that is affected by the report's event. | keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Rules @@ -745,7 +742,6 @@ An example event for `rules` looks as following: | google_workspace.rules.update_time_usec | Update time (microseconds since epoch) indicating the version of rule which is used. | date | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Admin @@ -977,7 +973,6 @@ An example event for `admin` looks as following: | google_workspace.organization.domain | The domain that is affected by the report's event. | keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Drive @@ -1145,7 +1140,6 @@ An example event for `drive` looks as following: | google_workspace.organization.domain | The domain that is affected by the report's event. | keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Groups @@ -1300,7 +1294,6 @@ An example event for `groups` looks as following: | google_workspace.organization.domain | The domain that is affected by the report's event. | keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Alert @@ -1617,7 +1610,6 @@ An example event for `alert` looks as following: | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | User defined tags. | keyword | ### Device @@ -1877,7 +1869,6 @@ An example event for `device` looks as following: | google_workspace.organization.domain | The domain that is affected by the report's event. | keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Group Enterprise @@ -2056,7 +2047,6 @@ An example event for `group_enterprise` looks as following: | google_workspace.organization.domain | The domain that is affected by the report's event. | keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Token @@ -2264,7 +2254,6 @@ An example event for `token` looks as following: | google_workspace.token.scope.value | Scopes under which access was granted / revoked. | keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Access Transparency @@ -2435,7 +2424,6 @@ An example event for `access_transparency` looks as following: | google_workspace.organization.domain | The domain that is affected by the report's event. | keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### Context Aware Access @@ -2595,7 +2583,6 @@ An example event for `context_aware_access` looks as following: | google_workspace.organization.domain | The domain that is affected by the report's event. | keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | ### GCP @@ -2741,5 +2728,4 @@ An example event for `gcp` looks as following: | google_workspace.organization.domain | The domain that is affected by the report's event. | keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | diff --git a/packages/google_workspace/manifest.yml b/packages/google_workspace/manifest.yml index df1eafee766..2f4eef163c6 100644 --- a/packages/google_workspace/manifest.yml +++ b/packages/google_workspace/manifest.yml @@ -1,6 +1,6 @@ name: google_workspace title: Google Workspace -version: "2.22.0" +version: "2.23.0" source: license: Elastic-2.0 description: Collect logs from Google Workspace with Elastic Agent. @@ -11,7 +11,7 @@ categories: - productivity_security conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: subscription: basic screenshots: From afa38a8c27672047ad5588b311b5a0f7bf3a9cc7 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:25:13 +0930 Subject: [PATCH 049/121] [http_endpoint] - Updated fields definitions Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. The ecs.version in sample_event.json files was changed to 8.11.0. Previously sample_event.json files contained 8.0.0. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/http_endpoint --- packages/http_endpoint/changelog.yml | 5 +++++ packages/http_endpoint/fields/agent.yml | 3 --- packages/http_endpoint/fields/beats.yml | 3 --- packages/http_endpoint/fields/ecs.yml | 12 ------------ packages/http_endpoint/manifest.yml | 2 +- packages/http_endpoint/sample_event.json | 2 +- 6 files changed, 7 insertions(+), 20 deletions(-) delete mode 100644 packages/http_endpoint/fields/ecs.yml diff --git a/packages/http_endpoint/changelog.yml b/packages/http_endpoint/changelog.yml index 942917443fe..9a2a2593ea9 100644 --- a/packages/http_endpoint/changelog.yml +++ b/packages/http_endpoint/changelog.yml @@ -1,3 +1,8 @@ +- version: "2.2.0" + changes: + - description: ECS version updated to 8.11.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.1.0" changes: - description: Provide request tracing support. diff --git a/packages/http_endpoint/fields/agent.yml b/packages/http_endpoint/fields/agent.yml index 230f7bc911d..9638d2992eb 100644 --- a/packages/http_endpoint/fields/agent.yml +++ b/packages/http_endpoint/fields/agent.yml @@ -5,9 +5,6 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. diff --git a/packages/http_endpoint/fields/beats.yml b/packages/http_endpoint/fields/beats.yml index ede69588554..22565288a1d 100644 --- a/packages/http_endpoint/fields/beats.yml +++ b/packages/http_endpoint/fields/beats.yml @@ -1,6 +1,3 @@ - name: input.type description: Type of Filebeat input. type: keyword -- name: tags - type: keyword - description: User defined tags diff --git a/packages/http_endpoint/fields/ecs.yml b/packages/http_endpoint/fields/ecs.yml deleted file mode 100644 index 21845b26f5a..00000000000 --- a/packages/http_endpoint/fields/ecs.yml +++ /dev/null @@ -1,12 +0,0 @@ -- name: ecs.version - external: ecs -- name: event.code - external: ecs -- name: event.created - external: ecs -- name: log.level - external: ecs -- name: message - external: ecs -- name: event.original - external: ecs diff --git a/packages/http_endpoint/manifest.yml b/packages/http_endpoint/manifest.yml index 3801af9426b..d79ff30a9fd 100644 --- a/packages/http_endpoint/manifest.yml +++ b/packages/http_endpoint/manifest.yml @@ -3,7 +3,7 @@ name: http_endpoint title: Custom HTTP Endpoint Logs description: Collect JSON data from listening HTTP port with Elastic Agent. type: input -version: "2.1.0" +version: "2.2.0" conditions: kibana: version: "^8.14.0" diff --git a/packages/http_endpoint/sample_event.json b/packages/http_endpoint/sample_event.json index a5e5c3ebf27..14494c99749 100644 --- a/packages/http_endpoint/sample_event.json +++ b/packages/http_endpoint/sample_event.json @@ -13,7 +13,7 @@ "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "11a5b254-bd1f-402d-9d5c-593cbebda407", From 9ec255136221967aee6391a550b1625805ddfd4d Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:25:13 +0930 Subject: [PATCH 050/121] [httpjson] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/httpjson --- packages/httpjson/changelog.yml | 5 +++++ packages/httpjson/data_stream/generic/fields/beats.yml | 3 --- packages/httpjson/data_stream/generic/fields/ecs.yml | 6 ------ packages/httpjson/manifest.yml | 4 ++-- 4 files changed, 7 insertions(+), 11 deletions(-) delete mode 100644 packages/httpjson/data_stream/generic/fields/ecs.yml diff --git a/packages/httpjson/changelog.yml b/packages/httpjson/changelog.yml index fb16a20021b..0ec7bd5e568 100644 --- a/packages/httpjson/changelog.yml +++ b/packages/httpjson/changelog.yml @@ -1,3 +1,8 @@ +- version: "1.21.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.20.0" changes: - description: Set sensitive values as secret. diff --git a/packages/httpjson/data_stream/generic/fields/beats.yml b/packages/httpjson/data_stream/generic/fields/beats.yml index ede69588554..22565288a1d 100644 --- a/packages/httpjson/data_stream/generic/fields/beats.yml +++ b/packages/httpjson/data_stream/generic/fields/beats.yml @@ -1,6 +1,3 @@ - name: input.type description: Type of Filebeat input. type: keyword -- name: tags - type: keyword - description: User defined tags diff --git a/packages/httpjson/data_stream/generic/fields/ecs.yml b/packages/httpjson/data_stream/generic/fields/ecs.yml deleted file mode 100644 index 1c3645d5f4e..00000000000 --- a/packages/httpjson/data_stream/generic/fields/ecs.yml +++ /dev/null @@ -1,6 +0,0 @@ -- name: ecs.version - external: ecs -- name: event.created - external: ecs -- name: message - external: ecs diff --git a/packages/httpjson/manifest.yml b/packages/httpjson/manifest.yml index a58c6214b82..38c5ce4f2d9 100644 --- a/packages/httpjson/manifest.yml +++ b/packages/httpjson/manifest.yml @@ -3,10 +3,10 @@ name: httpjson title: Custom API description: Collect custom events from an API endpoint with Elastic agent type: integration -version: "1.20.0" +version: "1.21.0" conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" categories: - custom policy_templates: From 87d7248363ebb8116c71137fbbf4af90da965c70 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:25:14 +0930 Subject: [PATCH 051/121] [imperva_cloud_waf] - Updated fields definitions Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/imperva_cloud_waf --- packages/imperva_cloud_waf/changelog.yml | 5 +++++ .../event/_dev/test/pipeline/test-common-config.yml | 1 - .../imperva_cloud_waf/data_stream/event/fields/beats.yml | 3 --- packages/imperva_cloud_waf/docs/README.md | 1 - packages/imperva_cloud_waf/manifest.yml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/imperva_cloud_waf/changelog.yml b/packages/imperva_cloud_waf/changelog.yml index 247b1d90ee3..5e79d16a91c 100644 --- a/packages/imperva_cloud_waf/changelog.yml +++ b/packages/imperva_cloud_waf/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.1.0" + changes: + - description: Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.0.0" changes: - description: Release package as GA. diff --git a/packages/imperva_cloud_waf/data_stream/event/_dev/test/pipeline/test-common-config.yml b/packages/imperva_cloud_waf/data_stream/event/_dev/test/pipeline/test-common-config.yml index 36106b22efb..1f0a54d166d 100644 --- a/packages/imperva_cloud_waf/data_stream/event/_dev/test/pipeline/test-common-config.yml +++ b/packages/imperva_cloud_waf/data_stream/event/_dev/test/pipeline/test-common-config.yml @@ -2,7 +2,6 @@ fields: tags: - preserve_original_event - preserve_duplicate_custom_fields - dynamic_fields: # This can be removed after ES 8.14 is the minimum version. # Relates: https://github.com/elastic/elasticsearch/pull/105689 diff --git a/packages/imperva_cloud_waf/data_stream/event/fields/beats.yml b/packages/imperva_cloud_waf/data_stream/event/fields/beats.yml index 083dcfe307e..fff1b3f1b6b 100644 --- a/packages/imperva_cloud_waf/data_stream/event/fields/beats.yml +++ b/packages/imperva_cloud_waf/data_stream/event/fields/beats.yml @@ -4,9 +4,6 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. - name: aws.s3 type: group fields: diff --git a/packages/imperva_cloud_waf/docs/README.md b/packages/imperva_cloud_waf/docs/README.md index 7804458cb4b..ff9072a365b 100644 --- a/packages/imperva_cloud_waf/docs/README.md +++ b/packages/imperva_cloud_waf/docs/README.md @@ -356,5 +356,4 @@ An example event for `event` looks as following: | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | | source.service.name | | keyword | -| tags | User defined tags. | keyword | diff --git a/packages/imperva_cloud_waf/manifest.yml b/packages/imperva_cloud_waf/manifest.yml index 234b126935e..b63e836fcac 100644 --- a/packages/imperva_cloud_waf/manifest.yml +++ b/packages/imperva_cloud_waf/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.3 name: imperva_cloud_waf title: Imperva Cloud WAF -version: 1.0.0 +version: "1.1.0" description: Collect logs from Imperva Cloud WAF with Elastic Agent. type: integration categories: From aabe34a7dddc88c5e641220cbb87a1fe931043cc Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:25:16 +0930 Subject: [PATCH 052/121] [infoblox_bloxone_ddi] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/infoblox_bloxone_ddi --- packages/infoblox_bloxone_ddi/changelog.yml | 5 + .../data_stream/dhcp_lease/fields/agent.yml | 137 ---------------- .../data_stream/dhcp_lease/fields/ecs.yml | 30 ---- .../data_stream/dns_config/fields/agent.yml | 147 ------------------ .../data_stream/dns_config/fields/ecs.yml | 22 --- .../data_stream/dns_data/fields/agent.yml | 147 ------------------ .../data_stream/dns_data/fields/ecs.yml | 37 ----- packages/infoblox_bloxone_ddi/docs/README.md | 120 -------------- packages/infoblox_bloxone_ddi/manifest.yml | 4 +- 9 files changed, 7 insertions(+), 642 deletions(-) delete mode 100644 packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/fields/ecs.yml delete mode 100644 packages/infoblox_bloxone_ddi/data_stream/dns_config/fields/ecs.yml delete mode 100644 packages/infoblox_bloxone_ddi/data_stream/dns_data/fields/ecs.yml diff --git a/packages/infoblox_bloxone_ddi/changelog.yml b/packages/infoblox_bloxone_ddi/changelog.yml index d501255c179..33bf888e70f 100644 --- a/packages/infoblox_bloxone_ddi/changelog.yml +++ b/packages/infoblox_bloxone_ddi/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.18.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.17.0" changes: - description: Improve handling of empty responses. diff --git a/packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/fields/agent.yml b/packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/fields/agent.yml index 13b5e5c01c0..894e6f12be2 100644 --- a/packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/fields/agent.yml +++ b/packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/fields/agent.yml @@ -5,152 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/fields/ecs.yml b/packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/fields/ecs.yml deleted file mode 100644 index 9e89db6b837..00000000000 --- a/packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/fields/ecs.yml +++ /dev/null @@ -1,30 +0,0 @@ -- external: ecs - name: client.user.id -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.end -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.start -- external: ecs - name: event.type -- external: ecs - name: host.hostname -- external: ecs - name: host.name -- external: ecs - name: network.type -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: tags diff --git a/packages/infoblox_bloxone_ddi/data_stream/dns_config/fields/agent.yml b/packages/infoblox_bloxone_ddi/data_stream/dns_config/fields/agent.yml index 6e1bac042bc..894e6f12be2 100644 --- a/packages/infoblox_bloxone_ddi/data_stream/dns_config/fields/agent.yml +++ b/packages/infoblox_bloxone_ddi/data_stream/dns_config/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/infoblox_bloxone_ddi/data_stream/dns_config/fields/ecs.yml b/packages/infoblox_bloxone_ddi/data_stream/dns_config/fields/ecs.yml deleted file mode 100644 index 3d366d56904..00000000000 --- a/packages/infoblox_bloxone_ddi/data_stream/dns_config/fields/ecs.yml +++ /dev/null @@ -1,22 +0,0 @@ -- external: ecs - name: dns.answers.ttl -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: related.hash -- external: ecs - name: related.ip -- external: ecs - name: tags diff --git a/packages/infoblox_bloxone_ddi/data_stream/dns_data/fields/agent.yml b/packages/infoblox_bloxone_ddi/data_stream/dns_data/fields/agent.yml index 6e1bac042bc..894e6f12be2 100644 --- a/packages/infoblox_bloxone_ddi/data_stream/dns_data/fields/agent.yml +++ b/packages/infoblox_bloxone_ddi/data_stream/dns_data/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/infoblox_bloxone_ddi/data_stream/dns_data/fields/ecs.yml b/packages/infoblox_bloxone_ddi/data_stream/dns_data/fields/ecs.yml deleted file mode 100644 index 4ef70f75af8..00000000000 --- a/packages/infoblox_bloxone_ddi/data_stream/dns_data/fields/ecs.yml +++ /dev/null @@ -1,37 +0,0 @@ -- external: ecs - name: dns.answers - type: group -- external: ecs - name: dns.answers.data -- external: ecs - name: dns.answers.ttl -- external: ecs - name: dns.answers.type -- external: ecs - name: dns.question.name -- external: ecs - name: dns.question.registered_domain -- external: ecs - name: dns.question.subdomain -- external: ecs - name: dns.question.type -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: tags diff --git a/packages/infoblox_bloxone_ddi/docs/README.md b/packages/infoblox_bloxone_ddi/docs/README.md index 982b96a446b..6120e87b5e4 100644 --- a/packages/infoblox_bloxone_ddi/docs/README.md +++ b/packages/infoblox_bloxone_ddi/docs/README.md @@ -156,50 +156,15 @@ An example event for `dhcp_lease` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| client.user.id | Unique identifier of the user. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | infoblox_bloxone_ddi.dhcp_lease.address | The IP address of the DHCP lease in the format "a.b.c.d". This address will be marked as leased in IPAM while the lease exists. | ip | | infoblox_bloxone_ddi.dhcp_lease.client_id | The client ID of the DHCP lease. It might be empty. | keyword | | infoblox_bloxone_ddi.dhcp_lease.ends | The time when the DHCP lease will expire. | date | @@ -220,10 +185,6 @@ An example event for `dhcp_lease` looks as following: | infoblox_bloxone_ddi.dhcp_lease.type | Lease type. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| tags | List of keywords used to tag each event. | keyword | ### dns_config @@ -913,49 +874,15 @@ An example event for `dns_config` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| dns.answers.ttl | The time interval in seconds that this resource record may be cached before it should be discarded. Zero values mean that the data should not be cached. | long | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | infoblox_bloxone_ddi.dns_config.add_edns.option_in.outgoing_query | add_edns_option_in_outgoing_query adds client IP, MAC address and view name into outgoing recursive query. | boolean | | infoblox_bloxone_ddi.dns_config.comment | Optional. Comment for view. | keyword | | infoblox_bloxone_ddi.dns_config.created_at | The timestamp when the object has been created. | date | @@ -1250,9 +1177,6 @@ An example event for `dns_config` looks as following: | infoblox_bloxone_ddi.dns_config.zone_authority.use_default_mname | Optional. Use default value for master name server. Defaults to true. | boolean | | input.type | Input type | keyword | | log.offset | Log offset | long | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| tags | List of keywords used to tag each event. | keyword | ### dns_data @@ -1417,56 +1341,15 @@ An example event for `dns_data` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| dns.answers | An array containing an object for each answer section returned by the server. The main keys that should be present in these objects are defined by ECS. Records that have more information may contain more keys than what ECS defines. Not all DNS data sources give all details about DNS answers. At minimum, answer objects must contain the `data` key. If more information is available, map as much of it to ECS as possible, and add any additional fields to the answer objects as custom fields. | group | -| dns.answers.data | The data describing the resource. The meaning of this data depends on the type and class of the resource record. | keyword | -| dns.answers.ttl | The time interval in seconds that this resource record may be cached before it should be discarded. Zero values mean that the data should not be cached. | long | -| dns.answers.type | The type of data contained in this resource record. | keyword | -| dns.question.name | The name being queried. If the name field contains non-printable characters (below 32 or above 126), those characters should be represented as escaped base 10 integers (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, and line feeds should be converted to \t, \r, and \n respectively. | keyword | -| dns.question.registered_domain | The highest registered domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| dns.question.subdomain | The subdomain is all of the labels under the registered_domain. If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| dns.question.type | The type of record being queried. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | infoblox_bloxone_ddi.dns_data.absolute.name.spec | The DNS protocol textual representation of absolute_name_spec. | keyword | | infoblox_bloxone_ddi.dns_data.absolute.zone.name | The DNS protocol textual representation of the absolute domain name of the zone where this record belongs. | keyword | | infoblox_bloxone_ddi.dns_data.absolute_name.spec | Synthetic field, used to determine zone and/or name_in_zone field for records. | keyword | @@ -1524,7 +1407,4 @@ An example event for `dns_data` looks as following: | infoblox_bloxone_ddi.dns_data.zone | The resource identifier. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| tags | List of keywords used to tag each event. | keyword | diff --git a/packages/infoblox_bloxone_ddi/manifest.yml b/packages/infoblox_bloxone_ddi/manifest.yml index bd292e6a4d2..a22e0b99ce5 100644 --- a/packages/infoblox_bloxone_ddi/manifest.yml +++ b/packages/infoblox_bloxone_ddi/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: infoblox_bloxone_ddi title: Infoblox BloxOne DDI -version: "1.17.0" +version: "1.18.0" description: Collect logs from Infoblox BloxOne DDI with Elastic Agent. type: integration categories: @@ -10,7 +10,7 @@ categories: - dns_security conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" screenshots: - src: /img/infoblox-bloxone-ddi-screenshot.png title: Infoblox BloxOne DDI dashboard screenshot From 55e31481029d761dd25887f51b0b6c06d74e1fb9 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:25:18 +0930 Subject: [PATCH 053/121] [infoblox_nios] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.7.1 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/infoblox_nios --- packages/infoblox_nios/changelog.yml | 5 + .../data_stream/log/fields/agent.yml | 143 ------------------ .../data_stream/log/fields/ecs.yml | 90 ----------- packages/infoblox_nios/docs/README.md | 72 --------- packages/infoblox_nios/manifest.yml | 4 +- 5 files changed, 7 insertions(+), 307 deletions(-) delete mode 100644 packages/infoblox_nios/data_stream/log/fields/ecs.yml diff --git a/packages/infoblox_nios/changelog.yml b/packages/infoblox_nios/changelog.yml index 4e18ab2716a..0f6c058b106 100644 --- a/packages/infoblox_nios/changelog.yml +++ b/packages/infoblox_nios/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.23.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.22.0" changes: - description: Handle REFUSED log messages. diff --git a/packages/infoblox_nios/data_stream/log/fields/agent.yml b/packages/infoblox_nios/data_stream/log/fields/agent.yml index 152150fe41a..5b567f262ee 100644 --- a/packages/infoblox_nios/data_stream/log/fields/agent.yml +++ b/packages/infoblox_nios/data_stream/log/fields/agent.yml @@ -5,158 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: >- diff --git a/packages/infoblox_nios/data_stream/log/fields/ecs.yml b/packages/infoblox_nios/data_stream/log/fields/ecs.yml deleted file mode 100644 index dcd80b6b0e6..00000000000 --- a/packages/infoblox_nios/data_stream/log/fields/ecs.yml +++ /dev/null @@ -1,90 +0,0 @@ -- external: ecs - name: client.geo.city_name -- external: ecs - name: client.geo.country_name -- external: ecs - name: client.geo.country_iso_code -- external: ecs - name: client.geo.continent_name -- external: ecs - name: client.geo.region_iso_code -- external: ecs - name: client.geo.location -- external: ecs - name: client.geo.region_name -- external: ecs - name: client.as.number -- external: ecs - name: client.as.organization.name -- external: ecs - name: client.ip -- external: ecs - name: client.mac -- external: ecs - name: client.port -- external: ecs - name: dns.answers.class -- external: ecs - name: dns.answers.data -- external: ecs - name: dns.answers.name -- external: ecs - name: dns.answers.ttl -- external: ecs - name: dns.answers.type -- external: ecs - name: dns.header_flags -- external: ecs - name: dns.question.class -- external: ecs - name: dns.question.name -- external: ecs - name: dns.question.registered_domain -- external: ecs - name: dns.question.subdomain -- external: ecs - name: dns.question.top_level_domain -- external: ecs - name: dns.question.type -- external: ecs - name: dns.response_code -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.type -- external: ecs - name: host.ip -- external: ecs - name: observer.ingress.interface.name -- external: ecs - name: log.file.path -- external: ecs - name: log.syslog.priority -- external: ecs - name: message -- external: ecs - name: network.transport -- external: ecs - name: process.pid -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: server.ip -- external: ecs - name: server.port -- external: ecs - name: tags -- external: ecs - name: user.name diff --git a/packages/infoblox_nios/docs/README.md b/packages/infoblox_nios/docs/README.md index 1294714ddbd..151f359eeef 100644 --- a/packages/infoblox_nios/docs/README.md +++ b/packages/infoblox_nios/docs/README.md @@ -238,71 +238,13 @@ An example event for `log` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| client.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| client.as.organization.name | Organization name. | keyword | -| client.as.organization.name.text | Multi-field of `client.as.organization.name`. | match_only_text | -| client.geo.city_name | City name. | keyword | -| client.geo.continent_name | Name of the continent. | keyword | -| client.geo.country_iso_code | Country ISO code. | keyword | -| client.geo.country_name | Country name. | keyword | -| client.geo.location | Longitude and latitude. | geo_point | -| client.geo.region_iso_code | Region ISO code. | keyword | -| client.geo.region_name | Region name. | keyword | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| client.mac | MAC address of the client. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| client.port | Port of the client. | long | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| dns.answers.class | The class of DNS data contained in this resource record. | keyword | -| dns.answers.data | The data describing the resource. The meaning of this data depends on the type and class of the resource record. | keyword | -| dns.answers.name | The domain name to which this resource record pertains. If a chain of CNAME is being resolved, each answer's `name` should be the one that corresponds with the answer's `data`. It should not simply be the original `question.name` repeated. | keyword | -| dns.answers.ttl | The time interval in seconds that this resource record may be cached before it should be discarded. Zero values mean that the data should not be cached. | long | -| dns.answers.type | The type of data contained in this resource record. | keyword | -| dns.header_flags | Array of 2 letter DNS header flags. | keyword | -| dns.question.class | The class of records being queried. | keyword | -| dns.question.name | The name being queried. If the name field contains non-printable characters (below 32 or above 126), those characters should be represented as escaped base 10 integers (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, and line feeds should be converted to \t, \r, and \n respectively. | keyword | -| dns.question.registered_domain | The highest registered domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| dns.question.subdomain | The subdomain is all of the labels under the registered_domain. If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| dns.question.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| dns.question.type | The type of record being queried. | keyword | -| dns.response_code | The DNS response code. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | infoblox_nios.log.audit.apparently_via | | keyword | | infoblox_nios.log.audit.auth | | keyword | | infoblox_nios.log.audit.error | | text | @@ -359,20 +301,6 @@ An example event for `log` looks as following: | infoblox_nios.log.type | | keyword | | infoblox_nios.log.view | | keyword | | input.type | Input type | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | Log offset | long | | log.source.address | Log source address | keyword | -| log.syslog.priority | Syslog numeric priority of the event, if available. According to RFCs 5424 and 3164, the priority is 8 \* facility + severity. This number is therefore expected to contain a value between 0 and 191. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| observer.ingress.interface.name | Interface name as reported by the system. | keyword | -| process.pid | Process id. | long | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| server.ip | IP address of the server (IPv4 or IPv6). | ip | -| server.port | Port of the server. | long | -| tags | List of keywords used to tag each event. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | diff --git a/packages/infoblox_nios/manifest.yml b/packages/infoblox_nios/manifest.yml index dff733e7f50..0fe93053f71 100644 --- a/packages/infoblox_nios/manifest.yml +++ b/packages/infoblox_nios/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: infoblox_nios title: Infoblox NIOS -version: "1.22.0" +version: "1.23.0" description: Collect logs from Infoblox NIOS with Elastic Agent. type: integration categories: @@ -10,7 +10,7 @@ categories: - dns_security conditions: kibana: - version: ^8.7.1 + version: "^8.13.0" screenshots: - src: /img/infoblox-nios-screenshot.png title: Infoblox NIOS dashboard screenshot From e7e19da9639170af51531ad8c05aff0105934327 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:25:25 +0930 Subject: [PATCH 054/121] [jamf_compliance_reporter] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.7.1 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/jamf_compliance_reporter --- .../jamf_compliance_reporter/changelog.yml | 5 + .../data_stream/log/fields/agent.yml | 147 ------------------ .../data_stream/log/fields/ecs.yml | 78 ---------- .../jamf_compliance_reporter/docs/README.md | 71 --------- .../jamf_compliance_reporter/manifest.yml | 4 +- 5 files changed, 7 insertions(+), 298 deletions(-) delete mode 100644 packages/jamf_compliance_reporter/data_stream/log/fields/ecs.yml diff --git a/packages/jamf_compliance_reporter/changelog.yml b/packages/jamf_compliance_reporter/changelog.yml index d79e16252a3..7c1f82e2750 100644 --- a/packages/jamf_compliance_reporter/changelog.yml +++ b/packages/jamf_compliance_reporter/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.13.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.12.0" changes: - description: Update manifest format version to v3.0.3. diff --git a/packages/jamf_compliance_reporter/data_stream/log/fields/agent.yml b/packages/jamf_compliance_reporter/data_stream/log/fields/agent.yml index 2919f7a30c6..35ae17d33b3 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/fields/agent.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/fields/agent.yml @@ -5,162 +5,15 @@ footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: "The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier." - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime." - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: "A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: "Hostname of the host. It normally contains what the `hostname` command returns on the host machine." - - name: id - level: core - type: keyword - ignore_above: 1024 - description: "Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: "Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: "Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." - name: containerized type: boolean description: >- diff --git a/packages/jamf_compliance_reporter/data_stream/log/fields/ecs.yml b/packages/jamf_compliance_reporter/data_stream/log/fields/ecs.yml deleted file mode 100644 index c09ccff0735..00000000000 --- a/packages/jamf_compliance_reporter/data_stream/log/fields/ecs.yml +++ /dev/null @@ -1,78 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: error.code -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.code -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.type -- external: ecs - name: file.hash.sha1 -- external: ecs - name: file.path -- external: ecs - name: host.os.type -- external: ecs - name: process.args -- external: ecs - name: process.exit_code -- external: ecs - name: process.hash.sha1 -- external: ecs - name: process.name -- external: ecs - name: process.pid -- external: ecs - name: process.parent.pid -- external: ecs - name: process.real_group.id -- external: ecs - name: process.real_group.name -- external: ecs - name: process.real_user.id -- external: ecs - name: process.real_user.name -- external: ecs - name: process.user.id -- external: ecs - name: process.user.name -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: server.ip -- external: ecs - name: server.port -- external: ecs - name: related.user -- external: ecs - name: tags -- external: ecs - name: user.effective.id -- external: ecs - name: user.effective.name -- external: ecs - name: user.email -- external: ecs - name: user.group.id -- external: ecs - name: user.group.name -- external: ecs - name: user.id -- external: ecs - name: user.name diff --git a/packages/jamf_compliance_reporter/docs/README.md b/packages/jamf_compliance_reporter/docs/README.md index 69e513aa4e4..c9fbd9727a5 100644 --- a/packages/jamf_compliance_reporter/docs/README.md +++ b/packages/jamf_compliance_reporter/docs/README.md @@ -201,55 +201,15 @@ An example event for `log` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.code | Error code describing the error. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Name of the dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.hash.sha1 | SHA1 hash. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.type | Use the `os.type` field to categorize the operating system into one of the broad commercial families. If the OS you're dealing with is not listed as an expected value, the field should not be populated. Please let us know by opening an issue with ECS, to propose its addition. | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | jamf_compliance_reporter.log.app_metric_info.cpu_percentage | | double | | jamf_compliance_reporter.log.app_metric_info.cpu_time_seconds | | double | @@ -490,35 +450,4 @@ An example event for `log` looks as following: | jamf_compliance_reporter.log.texts | | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| process.args | Array of process arguments, starting with the absolute path to the executable. May be filtered to protect sensitive information. | keyword | -| process.exit_code | The exit code of the process, if this is a termination event. The field should be absent if there is no exit code for the event (e.g. process start). | long | -| process.hash.sha1 | SHA1 hash. | keyword | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.parent.pid | Process id. | long | -| process.pid | Process id. | long | -| process.real_group.id | Unique identifier for the group on the system/platform. | keyword | -| process.real_group.name | Name of the group. | keyword | -| process.real_user.id | Unique identifier of the user. | keyword | -| process.real_user.name | Short name or login of the user. | keyword | -| process.real_user.name.text | Multi-field of `process.real_user.name`. | match_only_text | -| process.user.id | Unique identifier of the user. | keyword | -| process.user.name | Short name or login of the user. | keyword | -| process.user.name.text | Multi-field of `process.user.name`. | match_only_text | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| server.ip | IP address of the server (IPv4 or IPv6). | ip | -| server.port | Port of the server. | long | -| tags | List of keywords used to tag each event. | keyword | -| user.effective.id | Unique identifier of the user. | keyword | -| user.effective.name | Short name or login of the user. | keyword | -| user.effective.name.text | Multi-field of `user.effective.name`. | match_only_text | -| user.email | User email address. | keyword | -| user.group.id | Unique identifier for the group on the system/platform. | keyword | -| user.group.name | Name of the group. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | diff --git a/packages/jamf_compliance_reporter/manifest.yml b/packages/jamf_compliance_reporter/manifest.yml index 88305aaa722..5a56b3ebb28 100644 --- a/packages/jamf_compliance_reporter/manifest.yml +++ b/packages/jamf_compliance_reporter/manifest.yml @@ -1,14 +1,14 @@ format_version: "3.0.3" name: jamf_compliance_reporter title: Jamf Compliance Reporter -version: "1.12.0" +version: "1.13.0" description: Collect logs from Jamf Compliance Reporter with Elastic Agent. type: integration categories: - security conditions: kibana: - version: ^8.7.1 + version: "^8.13.0" screenshots: - src: /img/jamf-compliance-reporter-screenshot.png title: Jamf Compliance Reporter Screenshot From d1cb76d0e832bc815d5aeed6f5fe9e11f9dca7dd Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:25:38 +0930 Subject: [PATCH 055/121] [jamf_protect] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/jamf_protect --- packages/jamf_protect/changelog.yml | 5 + .../data_stream/alerts/fields/agent.yml | 147 ----- .../data_stream/alerts/fields/ecs.yml | 232 ------- .../data_stream/telemetry/fields/agent.yml | 147 ----- .../data_stream/telemetry/fields/ecs.yml | 256 -------- .../telemetry_legacy/fields/agent.yml | 147 ----- .../telemetry_legacy/fields/ecs.yml | 84 --- .../web_threat_events/fields/agent.yml | 147 ----- .../web_threat_events/fields/ecs.yml | 214 ------- .../web_traffic_events/fields/agent.yml | 147 ----- .../web_traffic_events/fields/ecs.yml | 166 ----- packages/jamf_protect/docs/README.md | 573 ------------------ packages/jamf_protect/manifest.yml | 4 +- 13 files changed, 7 insertions(+), 2262 deletions(-) delete mode 100644 packages/jamf_protect/data_stream/telemetry_legacy/fields/ecs.yml delete mode 100644 packages/jamf_protect/data_stream/web_traffic_events/fields/ecs.yml diff --git a/packages/jamf_protect/changelog.yml b/packages/jamf_protect/changelog.yml index 9d89330ef53..7b0f0049284 100644 --- a/packages/jamf_protect/changelog.yml +++ b/packages/jamf_protect/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.1.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.0.0" changes: - description: Adding support for new Telemetry stream. diff --git a/packages/jamf_protect/data_stream/alerts/fields/agent.yml b/packages/jamf_protect/data_stream/alerts/fields/agent.yml index 2919f7a30c6..35ae17d33b3 100644 --- a/packages/jamf_protect/data_stream/alerts/fields/agent.yml +++ b/packages/jamf_protect/data_stream/alerts/fields/agent.yml @@ -5,162 +5,15 @@ footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: "The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier." - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime." - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: "A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: "Hostname of the host. It normally contains what the `hostname` command returns on the host machine." - - name: id - level: core - type: keyword - ignore_above: 1024 - description: "Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: "Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: "Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." - name: containerized type: boolean description: >- diff --git a/packages/jamf_protect/data_stream/alerts/fields/ecs.yml b/packages/jamf_protect/data_stream/alerts/fields/ecs.yml index ed4a6adbfaf..9de37950ad1 100644 --- a/packages/jamf_protect/data_stream/alerts/fields/ecs.yml +++ b/packages/jamf_protect/data_stream/alerts/fields/ecs.yml @@ -1,235 +1,3 @@ -- external: ecs - name: container.image.tag -- external: ecs - name: container.runtime -- external: ecs - name: destination.address -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.domain -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.code -- external: ecs - name: event.created -- external: ecs - name: event.duration -- external: ecs - name: event.end -- external: ecs - name: event.id -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.provider -- external: ecs - name: event.severity -- external: ecs - name: event.start -- external: ecs - name: event.timezone -- external: ecs - name: event.type -- external: ecs - name: file.code_signature.signing_id -- external: ecs - name: file.code_signature.status -- external: ecs - name: file.code_signature.team_id -- external: ecs - name: file.extension -- external: ecs - name: file.gid -- external: ecs - name: file.hash.md5 -- external: ecs - name: file.hash.sha1 -- external: ecs - name: file.hash.sha256 -- external: ecs - name: file.hash.sha512 -- external: ecs - name: file.inode -- external: ecs - name: file.mode -- external: ecs - name: file.name -- external: ecs - name: file.path -- external: ecs - name: file.size -- external: ecs - name: file.uid -- external: ecs - name: group.id -- external: ecs - name: group.name -- external: ecs - name: log.file.path -- external: ecs - name: log.logger -- external: ecs - name: message -- external: ecs - name: network.direction -- external: ecs - name: network.transport -- external: ecs - name: observer.name -- external: ecs - name: observer.product -- external: ecs - name: observer.type -- external: ecs - name: observer.vendor -- external: ecs - name: process.args -- external: ecs - name: process.args_count -- external: ecs - name: process.code_signature.signing_id -- external: ecs - name: process.code_signature.status -- external: ecs - name: process.code_signature.team_id -- external: ecs - name: process.command_line -- external: ecs - name: process.entity_id -- external: ecs - name: process.executable -- external: ecs - name: process.exit_code -- external: ecs - name: process.group_leader.executable -- external: ecs - name: process.group_leader.group.id -- external: ecs - name: process.group_leader.name -- external: ecs - name: process.group_leader.pid -- external: ecs - name: process.group_leader.real_group.id -- external: ecs - name: process.group_leader.real_user.id -- external: ecs - name: process.group_leader.start -- external: ecs - name: process.group_leader.user.id -- external: ecs - name: process.hash.md5 -- external: ecs - name: process.hash.sha1 -- external: ecs - name: process.hash.sha256 -- external: ecs - name: process.name -- external: ecs - name: process.parent.code_signature.signing_id -- external: ecs - name: process.parent.code_signature.status -- external: ecs - name: process.parent.code_signature.team_id -- external: ecs - name: process.parent.entity_id -- external: ecs - name: process.parent.executable -- external: ecs - name: process.parent.name -- external: ecs - name: process.parent.pid -- external: ecs - name: process.parent.real_group.id -- external: ecs - name: process.parent.real_user.id -- external: ecs - name: process.parent.start -- external: ecs - name: process.parent.user.id -- external: ecs - name: process.pid -- external: ecs - name: process.real_group.id -- external: ecs - name: process.real_user.id -- external: ecs - name: process.start -- external: ecs - name: process.user.id -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: rule.description -- external: ecs - name: rule.name -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: tags -- external: ecs - name: threat.enrichments -- external: ecs - name: threat.framework -- external: ecs - name: threat.software.platforms -- external: ecs - name: threat.tactic.id -- external: ecs - name: threat.tactic.name -- external: ecs - name: threat.tactic.reference -- external: ecs - name: threat.technique.id -- external: ecs - name: threat.technique.name -- external: ecs - name: threat.technique.reference -- external: ecs - name: user.domain -- external: ecs - name: user.email -- external: ecs - name: user.id -- external: ecs - name: user.name - name: volume.file_system_type type: keyword - name: volume.bus_type diff --git a/packages/jamf_protect/data_stream/telemetry/fields/agent.yml b/packages/jamf_protect/data_stream/telemetry/fields/agent.yml index 2919f7a30c6..35ae17d33b3 100644 --- a/packages/jamf_protect/data_stream/telemetry/fields/agent.yml +++ b/packages/jamf_protect/data_stream/telemetry/fields/agent.yml @@ -5,162 +5,15 @@ footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: "The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier." - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime." - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: "A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: "Hostname of the host. It normally contains what the `hostname` command returns on the host machine." - - name: id - level: core - type: keyword - ignore_above: 1024 - description: "Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: "Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: "Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." - name: containerized type: boolean description: >- diff --git a/packages/jamf_protect/data_stream/telemetry/fields/ecs.yml b/packages/jamf_protect/data_stream/telemetry/fields/ecs.yml index b1a023ec96c..d1fb054c432 100644 --- a/packages/jamf_protect/data_stream/telemetry/fields/ecs.yml +++ b/packages/jamf_protect/data_stream/telemetry/fields/ecs.yml @@ -1,259 +1,3 @@ -- external: ecs - name: observer.version -- external: ecs - name: device.id -- external: ecs - name: device.manufacturer -- external: ecs - name: process.env_vars -- external: ecs - name: process.interactive -- external: ecs - name: process.thread.id -- external: ecs - name: event.action -- external: ecs - name: event.created -- external: ecs - name: event.code -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.severity -- external: ecs - name: event.start -- external: ecs - name: event.category -- external: ecs - name: event.id -- external: ecs - name: event.timezone -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: user.name -- external: ecs - name: user.id -- external: ecs - name: user.effective.id -- external: ecs - name: user.domain -- external: ecs - name: user.email -- external: ecs - name: related.hosts -- external: ecs - name: related.hash -- external: ecs - name: process.args -- external: ecs - name: process.working_directory -- external: ecs - name: process.args_count -- external: ecs - name: process.executable -- external: ecs - name: process.parent.pid -- external: ecs - name: process.group_leader.group.id -- external: ecs - name: process.real_group.id -- external: ecs - name: process.parent.real_group.id -- external: ecs - name: process.group_leader.real_group.id -- external: ecs - name: process.entity_id -- external: ecs - name: process.real_user.id -- external: ecs - name: process.parent.real_user.id -- external: ecs - name: process.group_leader.real_user.id -- external: ecs - name: process.user.id -- external: ecs - name: process.parent.user.id -- external: ecs - name: process.group_leader.user.id -- external: ecs - name: process.group_leader.pid -- external: ecs - name: process.exit_code -- external: ecs - name: process.name -- external: ecs - name: process.pid -- external: ecs - name: process.hash.md5 -- external: ecs - name: process.hash.sha1 -- external: ecs - name: process.hash.sha256 -- external: ecs - name: process.code_signature.signing_id -- external: ecs - name: process.code_signature.status -- external: ecs - name: process.code_signature.team_id -- external: ecs - name: file.hash.md5 -- external: ecs - name: file.hash.sha1 -- external: ecs - name: file.hash.sha256 -- external: ecs - name: file.name -- external: ecs - name: file.path -- external: ecs - name: file.gid -- external: ecs - name: file.inode -- external: ecs - name: file.mode -- external: ecs - name: file.size -- external: ecs - name: file.uid -- external: ecs - name: file.code_signature.signing_id -- external: ecs - name: file.code_signature.status -- external: ecs - name: file.code_signature.team_id -- external: ecs - name: destination.address -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.domain -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: network.direction -- external: ecs - name: network.transport -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: tags -- external: ecs - name: threat.tactic.id -- external: ecs - name: threat.tactic.reference -- external: ecs - name: threat.tactic.name -- external: ecs - name: threat.technique.id -- external: ecs - name: threat.technique.name -- external: ecs - name: threat.technique.reference -- external: ecs - name: threat.enrichments -- external: ecs - name: threat.software.platforms -- external: ecs - name: threat.indicator.file.path -- external: ecs - name: threat.indicator.type -- external: ecs - name: rule.version -- external: ecs - name: container.image.tag -- external: ecs - name: container.runtime -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.duration -- external: ecs - name: event.end -- external: ecs - name: event.provider -- external: ecs - name: event.type -- external: ecs - name: file.extension -- external: ecs - name: file.hash.sha512 -- external: ecs - name: log.file.path -- external: ecs - name: log.logger -- external: ecs - name: message -- external: ecs - name: observer.name -- external: ecs - name: observer.product -- external: ecs - name: observer.type -- external: ecs - name: observer.vendor -- external: ecs - name: process.command_line -- external: ecs - name: process.parent.name -- external: ecs - name: process.parent.executable -- external: ecs - name: process.parent.entity_id -- external: ecs - name: process.parent.start -- external: ecs - name: process.parent.code_signature.signing_id -- external: ecs - name: process.parent.code_signature.status -- external: ecs - name: process.parent.code_signature.team_id -- external: ecs - name: process.group_leader.name -- external: ecs - name: process.group_leader.executable -- external: ecs - name: process.group_leader.start -- external: ecs - name: process.group_leader.entity_id -- external: ecs - name: process.start -- external: ecs - name: rule.description -- external: ecs - name: rule.name -- external: ecs - name: threat.framework -- external: ecs - name: group.id -- external: ecs - name: group.name - name: volume.device_name type: keyword - name: volume.mount_name diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/fields/agent.yml b/packages/jamf_protect/data_stream/telemetry_legacy/fields/agent.yml index 2919f7a30c6..35ae17d33b3 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/fields/agent.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/fields/agent.yml @@ -5,162 +5,15 @@ footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: "The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier." - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime." - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: "A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: "Hostname of the host. It normally contains what the `hostname` command returns on the host machine." - - name: id - level: core - type: keyword - ignore_above: 1024 - description: "Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: "Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: "Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." - name: containerized type: boolean description: >- diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/fields/ecs.yml b/packages/jamf_protect/data_stream/telemetry_legacy/fields/ecs.yml deleted file mode 100644 index 6804143d1fe..00000000000 --- a/packages/jamf_protect/data_stream/telemetry_legacy/fields/ecs.yml +++ /dev/null @@ -1,84 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: error.code -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.code -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.outcome -- external: ecs - name: event.type -- external: ecs - name: file.hash.sha1 -- external: ecs - name: file.path -- external: ecs - name: host.os.type -- external: ecs - name: process.args -- external: ecs - name: process.exit_code -- external: ecs - name: process.name -- external: ecs - name: process.pid -- external: ecs - name: process.parent.pid -- external: ecs - name: process.hash.sha1 -- external: ecs - name: process.hash.sha256 -- external: ecs - name: process.real_group.id -- external: ecs - name: process.real_group.name -- external: ecs - name: process.real_user.id -- external: ecs - name: process.real_user.name -- external: ecs - name: process.user.id -- external: ecs - name: process.user.name -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: server.ip -- external: ecs - name: server.port -- external: ecs - name: related.user -- external: ecs - name: tags -- external: ecs - name: user.effective.id -- external: ecs - name: user.effective.name -- external: ecs - name: user.email -- external: ecs - name: user.group.id -- external: ecs - name: user.group.name -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: process.code_signature.signing_id -- external: ecs - name: process.code_signature.status -- external: ecs - name: process.code_signature.team_id diff --git a/packages/jamf_protect/data_stream/web_threat_events/fields/agent.yml b/packages/jamf_protect/data_stream/web_threat_events/fields/agent.yml index 2919f7a30c6..35ae17d33b3 100644 --- a/packages/jamf_protect/data_stream/web_threat_events/fields/agent.yml +++ b/packages/jamf_protect/data_stream/web_threat_events/fields/agent.yml @@ -5,162 +5,15 @@ footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: "The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier." - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime." - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: "A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: "Hostname of the host. It normally contains what the `hostname` command returns on the host machine." - - name: id - level: core - type: keyword - ignore_above: 1024 - description: "Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: "Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: "Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." - name: containerized type: boolean description: >- diff --git a/packages/jamf_protect/data_stream/web_threat_events/fields/ecs.yml b/packages/jamf_protect/data_stream/web_threat_events/fields/ecs.yml index 3e6112bc0e0..9de37950ad1 100644 --- a/packages/jamf_protect/data_stream/web_threat_events/fields/ecs.yml +++ b/packages/jamf_protect/data_stream/web_threat_events/fields/ecs.yml @@ -1,215 +1,3 @@ -- external: ecs - name: container.image.tag -- external: ecs - name: container.runtime -- external: ecs - name: destination.address -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.domain -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.code -- external: ecs - name: event.created -- external: ecs - name: event.duration -- external: ecs - name: event.end -- external: ecs - name: event.id -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.provider -- external: ecs - name: event.severity -- external: ecs - name: event.start -- external: ecs - name: event.timezone -- external: ecs - name: event.type -- external: ecs - name: file.code_signature.signing_id -- external: ecs - name: file.code_signature.status -- external: ecs - name: file.code_signature.team_id -- external: ecs - name: file.extension -- external: ecs - name: file.gid -- external: ecs - name: file.hash.md5 -- external: ecs - name: file.hash.sha1 -- external: ecs - name: file.hash.sha256 -- external: ecs - name: file.hash.sha512 -- external: ecs - name: file.inode -- external: ecs - name: file.mode -- external: ecs - name: file.name -- external: ecs - name: file.path -- external: ecs - name: file.size -- external: ecs - name: file.uid -- external: ecs - name: group.id -- external: ecs - name: group.name -- external: ecs - name: host.geo.country_iso_code -- external: ecs - name: log.file.path -- external: ecs - name: log.logger -- external: ecs - name: message -- external: ecs - name: network.direction -- external: ecs - name: network.transport -- external: ecs - name: observer.name -- external: ecs - name: observer.product -- external: ecs - name: observer.type -- external: ecs - name: observer.vendor -- external: ecs - name: process.args -- external: ecs - name: process.args_count -- external: ecs - name: process.code_signature.signing_id -- external: ecs - name: process.code_signature.status -- external: ecs - name: process.code_signature.team_id -- external: ecs - name: process.command_line -- external: ecs - name: process.executable -- external: ecs - name: process.exit_code -- external: ecs - name: process.group_leader.pid -- external: ecs - name: process.hash.md5 -- external: ecs - name: process.hash.sha1 -- external: ecs - name: process.hash.sha256 -- external: ecs - name: process.name -- external: ecs - name: process.parent.pid -- external: ecs - name: process.parent.start -- external: ecs - name: process.pid -- external: ecs - name: process.real_group.id -- external: ecs - name: process.real_user.id -- external: ecs - name: process.start -- external: ecs - name: process.tty.char_device.major -- external: ecs - name: process.tty.char_device.minor -- external: ecs - name: process.tty.columns -- external: ecs - name: process.tty.rows -- external: ecs - name: process.user.id -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: rule.description -- external: ecs - name: rule.id -- external: ecs - name: rule.name -- external: ecs - name: source.domain -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: tags -- external: ecs - name: threat.enrichments -- external: ecs - name: threat.framework -- external: ecs - name: threat.software.platforms -- external: ecs - name: threat.tactic.id -- external: ecs - name: threat.tactic.name -- external: ecs - name: threat.tactic.reference -- external: ecs - name: threat.technique.id -- external: ecs - name: threat.technique.name -- external: ecs - name: threat.technique.reference -- external: ecs - name: user.domain -- external: ecs - name: user.email -- external: ecs - name: user.id -- external: ecs - name: user.name - name: volume.file_system_type type: keyword - name: volume.bus_type @@ -232,5 +20,3 @@ type: keyword - name: volume.writable type: boolean -- name: organization.id - type: keyword diff --git a/packages/jamf_protect/data_stream/web_traffic_events/fields/agent.yml b/packages/jamf_protect/data_stream/web_traffic_events/fields/agent.yml index 2919f7a30c6..35ae17d33b3 100644 --- a/packages/jamf_protect/data_stream/web_traffic_events/fields/agent.yml +++ b/packages/jamf_protect/data_stream/web_traffic_events/fields/agent.yml @@ -5,162 +5,15 @@ footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: "The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier." - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime." - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: "A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: "Hostname of the host. It normally contains what the `hostname` command returns on the host machine." - - name: id - level: core - type: keyword - ignore_above: 1024 - description: "Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: "Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: "Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." - name: containerized type: boolean description: >- diff --git a/packages/jamf_protect/data_stream/web_traffic_events/fields/ecs.yml b/packages/jamf_protect/data_stream/web_traffic_events/fields/ecs.yml deleted file mode 100644 index 1ff3858a73a..00000000000 --- a/packages/jamf_protect/data_stream/web_traffic_events/fields/ecs.yml +++ /dev/null @@ -1,166 +0,0 @@ -- external: ecs - name: destination.address -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.domain -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: dns.answers.ttl -- external: ecs - name: dns.answers.type -- external: ecs - name: dns.question.name -- external: ecs - name: dns.question.registered_domain -- external: ecs - name: dns.question.top_level_domain -- external: ecs - name: dns.response_code -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.code -- external: ecs - name: event.created -- external: ecs - name: event.duration -- external: ecs - name: event.end -- external: ecs - name: event.id -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.provider -- external: ecs - name: event.severity -- external: ecs - name: event.start -- external: ecs - name: event.timezone -- external: ecs - name: event.type -- external: ecs - name: file.code_signature.signing_id -- external: ecs - name: file.code_signature.status -- external: ecs - name: file.code_signature.team_id -- external: ecs - name: file.extension -- external: ecs - name: file.gid -- external: ecs - name: file.hash.md5 -- external: ecs - name: file.hash.sha1 -- external: ecs - name: file.hash.sha256 -- external: ecs - name: file.hash.sha512 -- external: ecs - name: file.inode -- external: ecs - name: file.mode -- external: ecs - name: file.name -- external: ecs - name: file.path -- external: ecs - name: file.size -- external: ecs - name: file.uid -- external: ecs - name: group.id -- external: ecs - name: group.name -- external: ecs - name: host.geo.country_iso_code -- external: ecs - name: log.file.path -- external: ecs - name: log.logger -- external: ecs - name: message -- external: ecs - name: network.direction -- external: ecs - name: network.transport -- external: ecs - name: observer.name -- external: ecs - name: observer.product -- external: ecs - name: observer.type -- external: ecs - name: observer.vendor -- external: ecs - name: process.command_line -- external: ecs - name: process.parent.pid -- external: ecs - name: process.parent.start -- external: ecs - name: process.pid -- external: ecs - name: process.start -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: rule.description -- external: ecs - name: rule.id -- external: ecs - name: rule.name -- external: ecs - name: source.domain -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: tags -- external: ecs - name: user.domain -- external: ecs - name: user.email -- external: ecs - name: user.id -- external: ecs - name: user.name -- name: organization.id - type: keyword diff --git a/packages/jamf_protect/docs/README.md b/packages/jamf_protect/docs/README.md index a05b9804c29..f30f702bdd0 100644 --- a/packages/jamf_protect/docs/README.md +++ b/packages/jamf_protect/docs/README.md @@ -249,171 +249,18 @@ An example event for `alerts` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.image.tag | Container image tags. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| container.runtime | Runtime managing this container. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Name of the dataset. | constant_keyword | -| event.duration | Duration of the event in nanoseconds. If `event.start` and `event.end` are known this value should be the difference between the end and start time. | long | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.code_signature.signing_id | The identifier used to sign the process. This is used to identify the application manufactured by a software vendor. The field is relevant to Apple \*OS only. | keyword | -| file.code_signature.status | Additional information about the certificate status. This is useful for logging cryptographic errors with the certificate validity or trust status. Leave unpopulated if the validity or trust of the certificate was unchecked. | keyword | -| file.code_signature.team_id | The team identifier used to sign the process. This is used to identify the team or vendor of a software product. The field is relevant to Apple \*OS only. | keyword | -| file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| file.gid | Primary group ID (GID) of the file. | keyword | -| file.hash.md5 | MD5 hash. | keyword | -| file.hash.sha1 | SHA1 hash. | keyword | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.hash.sha512 | SHA512 hash. | keyword | -| file.inode | Inode representing the file in the filesystem. | keyword | -| file.mode | Mode of the file in octal representation. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| file.uid | The user ID (UID) or security identifier (SID) of the file owner. | keyword | -| group.id | Unique identifier for the group on the system/platform. | keyword | -| group.name | Name of the group. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | jamf_protect.alerts.timestamp_nanoseconds | The timestamp in Epoch nanoseconds. | date | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | -| log.logger | The name of the logger inside an application. This is usually the name of the class which initialized the logger, or can be a custom name. | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| observer.name | Custom name of the observer. This is a name that can be given to an observer. This can be helpful for example if multiple firewalls of the same model are used in an organization. If no custom name is needed, the field can be left empty. | keyword | -| observer.product | The product name of the observer. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| process.args | Array of process arguments, starting with the absolute path to the executable. May be filtered to protect sensitive information. | keyword | -| process.args_count | Length of the process.args array. This field can be useful for querying or performing bucket analysis on how many arguments were provided to start a process. More arguments may be an indication of suspicious activity. | long | -| process.code_signature.signing_id | The identifier used to sign the process. This is used to identify the application manufactured by a software vendor. The field is relevant to Apple \*OS only. | keyword | -| process.code_signature.status | Additional information about the certificate status. This is useful for logging cryptographic errors with the certificate validity or trust status. Leave unpopulated if the validity or trust of the certificate was unchecked. | keyword | -| process.code_signature.team_id | The team identifier used to sign the process. This is used to identify the team or vendor of a software product. The field is relevant to Apple \*OS only. | keyword | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.executable | Absolute path to the process executable. | keyword | -| process.executable.text | Multi-field of `process.executable`. | match_only_text | -| process.exit_code | The exit code of the process, if this is a termination event. The field should be absent if there is no exit code for the event (e.g. process start). | long | -| process.group_leader.executable | Absolute path to the process executable. | keyword | -| process.group_leader.executable.text | Multi-field of `process.group_leader.executable`. | match_only_text | -| process.group_leader.group.id | Unique identifier for the group on the system/platform. | keyword | -| process.group_leader.name | Process name. Sometimes called program name or similar. | keyword | -| process.group_leader.name.text | Multi-field of `process.group_leader.name`. | match_only_text | -| process.group_leader.pid | Process id. | long | -| process.group_leader.real_group.id | Unique identifier for the group on the system/platform. | keyword | -| process.group_leader.real_user.id | Unique identifier of the user. | keyword | -| process.group_leader.start | The time the process started. | date | -| process.group_leader.user.id | Unique identifier of the user. | keyword | -| process.hash.md5 | MD5 hash. | keyword | -| process.hash.sha1 | SHA1 hash. | keyword | -| process.hash.sha256 | SHA256 hash. | keyword | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.parent.code_signature.signing_id | The identifier used to sign the process. This is used to identify the application manufactured by a software vendor. The field is relevant to Apple \*OS only. | keyword | -| process.parent.code_signature.status | Additional information about the certificate status. This is useful for logging cryptographic errors with the certificate validity or trust status. Leave unpopulated if the validity or trust of the certificate was unchecked. | keyword | -| process.parent.code_signature.team_id | The team identifier used to sign the process. This is used to identify the team or vendor of a software product. The field is relevant to Apple \*OS only. | keyword | -| process.parent.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.parent.executable | Absolute path to the process executable. | keyword | -| process.parent.executable.text | Multi-field of `process.parent.executable`. | match_only_text | -| process.parent.name | Process name. Sometimes called program name or similar. | keyword | -| process.parent.name.text | Multi-field of `process.parent.name`. | match_only_text | -| process.parent.pid | Process id. | long | -| process.parent.real_group.id | Unique identifier for the group on the system/platform. | keyword | -| process.parent.real_user.id | Unique identifier of the user. | keyword | -| process.parent.start | The time the process started. | date | -| process.parent.user.id | Unique identifier of the user. | keyword | -| process.pid | Process id. | long | -| process.real_group.id | Unique identifier for the group on the system/platform. | keyword | -| process.real_user.id | Unique identifier of the user. | keyword | -| process.start | The time the process started. | date | -| process.user.id | Unique identifier of the user. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.description | The description of the rule generating the event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| threat.enrichments | A list of associated indicators objects enriching the event, and the context of that association/enrichment. | nested | -| threat.framework | Name of the threat framework used to further categorize and classify the tactic and technique of the reported threat. Framework classification can be provided by detecting systems, evaluated at ingest time, or retrospectively tagged to events. | keyword | -| threat.software.platforms | The platforms of the software used by this threat to conduct behavior commonly modeled using MITRE ATT&CK®. While not required, you can use MITRE ATT&CK® software platform values. | keyword | -| threat.tactic.id | The id of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/ ) | keyword | -| threat.tactic.name | Name of the type of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/) | keyword | -| threat.tactic.reference | The reference url of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/ ) | keyword | -| threat.technique.id | The id of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.name | The name of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.name.text | Multi-field of `threat.technique.name`. | match_only_text | -| threat.technique.reference | The reference url of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | | volume.bus_type | | keyword | | volume.file_system_type | | keyword | | volume.nt_name | | keyword | @@ -597,93 +444,15 @@ An example event for `telemetry` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.image.tag | Container image tags. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| container.runtime | Runtime managing this container. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| device.id | The unique identifier of a device. The identifier must not change across application sessions but stay fixed for an instance of a (mobile) device. On iOS, this value must be equal to the vendor identifier (https://developer.apple.com/documentation/uikit/uidevice/1620059-identifierforvendor). On Android, this value must be equal to the Firebase Installation ID or a globally unique UUID which is persisted across sessions in your application. For GDPR and data protection law reasons this identifier should not carry information that would allow to identify a user. | keyword | -| device.manufacturer | The vendor name of the device manufacturer. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Name of the dataset. | constant_keyword | -| event.duration | Duration of the event in nanoseconds. If `event.start` and `event.end` are known this value should be the difference between the end and start time. | long | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.code_signature.signing_id | The identifier used to sign the process. This is used to identify the application manufactured by a software vendor. The field is relevant to Apple \*OS only. | keyword | -| file.code_signature.status | Additional information about the certificate status. This is useful for logging cryptographic errors with the certificate validity or trust status. Leave unpopulated if the validity or trust of the certificate was unchecked. | keyword | -| file.code_signature.team_id | The team identifier used to sign the process. This is used to identify the team or vendor of a software product. The field is relevant to Apple \*OS only. | keyword | -| file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| file.gid | Primary group ID (GID) of the file. | keyword | -| file.hash.md5 | MD5 hash. | keyword | -| file.hash.sha1 | SHA1 hash. | keyword | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.hash.sha512 | SHA512 hash. | keyword | -| file.inode | Inode representing the file in the filesystem. | keyword | -| file.mode | Mode of the file in octal representation. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| file.uid | The user ID (UID) or security identifier (SID) of the file owner. | keyword | -| group.id | Unique identifier for the group on the system/platform. | keyword | -| group.name | Name of the group. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | jamf_protect.telemetry.account_type | Defines if it's a user or group | keyword | | jamf_protect.telemetry.attribute_name | The name of the attribute that got set | keyword | @@ -771,96 +540,7 @@ An example event for `telemetry` looks as following: | jamf_protect.telemetry.system_performance.timer_wakeups.wakeups | Number of wakeups | long | | jamf_protect.telemetry.to_username | Username to which an action is directed | keyword | | jamf_protect.telemetry.tty | Software terminal device file that the process is associated with | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | -| log.logger | The name of the logger inside an application. This is usually the name of the class which initialized the logger, or can be a custom name. | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| observer.name | Custom name of the observer. This is a name that can be given to an observer. This can be helpful for example if multiple firewalls of the same model are used in an organization. If no custom name is needed, the field can be left empty. | keyword | -| observer.product | The product name of the observer. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| observer.version | Observer version. | keyword | -| process.args | Array of process arguments, starting with the absolute path to the executable. May be filtered to protect sensitive information. | keyword | -| process.args_count | Length of the process.args array. This field can be useful for querying or performing bucket analysis on how many arguments were provided to start a process. More arguments may be an indication of suspicious activity. | long | -| process.code_signature.signing_id | The identifier used to sign the process. This is used to identify the application manufactured by a software vendor. The field is relevant to Apple \*OS only. | keyword | -| process.code_signature.status | Additional information about the certificate status. This is useful for logging cryptographic errors with the certificate validity or trust status. Leave unpopulated if the validity or trust of the certificate was unchecked. | keyword | -| process.code_signature.team_id | The team identifier used to sign the process. This is used to identify the team or vendor of a software product. The field is relevant to Apple \*OS only. | keyword | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.env_vars | Array of environment variable bindings. Captured from a snapshot of the environment at the time of execution. May be filtered to protect sensitive information. | keyword | -| process.executable | Absolute path to the process executable. | keyword | -| process.executable.text | Multi-field of `process.executable`. | match_only_text | -| process.exit_code | The exit code of the process, if this is a termination event. The field should be absent if there is no exit code for the event (e.g. process start). | long | -| process.group_leader.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.group_leader.executable | Absolute path to the process executable. | keyword | -| process.group_leader.executable.text | Multi-field of `process.group_leader.executable`. | match_only_text | -| process.group_leader.group.id | Unique identifier for the group on the system/platform. | keyword | -| process.group_leader.name | Process name. Sometimes called program name or similar. | keyword | -| process.group_leader.name.text | Multi-field of `process.group_leader.name`. | match_only_text | -| process.group_leader.pid | Process id. | long | -| process.group_leader.real_group.id | Unique identifier for the group on the system/platform. | keyword | -| process.group_leader.real_user.id | Unique identifier of the user. | keyword | -| process.group_leader.start | The time the process started. | date | -| process.group_leader.user.id | Unique identifier of the user. | keyword | -| process.hash.md5 | MD5 hash. | keyword | -| process.hash.sha1 | SHA1 hash. | keyword | -| process.hash.sha256 | SHA256 hash. | keyword | -| process.interactive | Whether the process is connected to an interactive shell. Process interactivity is inferred from the processes file descriptors. If the character device for the controlling tty is the same as stdin and stderr for the process, the process is considered interactive. Note: A non-interactive process can belong to an interactive session and is simply one that does not have open file descriptors reading the controlling TTY on FD 0 (stdin) or writing to the controlling TTY on FD 2 (stderr). A backgrounded process is still considered interactive if stdin and stderr are connected to the controlling TTY. | boolean | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.parent.code_signature.signing_id | The identifier used to sign the process. This is used to identify the application manufactured by a software vendor. The field is relevant to Apple \*OS only. | keyword | -| process.parent.code_signature.status | Additional information about the certificate status. This is useful for logging cryptographic errors with the certificate validity or trust status. Leave unpopulated if the validity or trust of the certificate was unchecked. | keyword | -| process.parent.code_signature.team_id | The team identifier used to sign the process. This is used to identify the team or vendor of a software product. The field is relevant to Apple \*OS only. | keyword | -| process.parent.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.parent.executable | Absolute path to the process executable. | keyword | -| process.parent.executable.text | Multi-field of `process.parent.executable`. | match_only_text | -| process.parent.name | Process name. Sometimes called program name or similar. | keyword | -| process.parent.name.text | Multi-field of `process.parent.name`. | match_only_text | -| process.parent.pid | Process id. | long | -| process.parent.real_group.id | Unique identifier for the group on the system/platform. | keyword | -| process.parent.real_user.id | Unique identifier of the user. | keyword | -| process.parent.start | The time the process started. | date | -| process.parent.user.id | Unique identifier of the user. | keyword | -| process.pid | Process id. | long | -| process.real_group.id | Unique identifier for the group on the system/platform. | keyword | -| process.real_user.id | Unique identifier of the user. | keyword | -| process.start | The time the process started. | date | -| process.thread.id | Thread ID. | long | -| process.user.id | Unique identifier of the user. | keyword | -| process.working_directory | The working directory of the process. | keyword | -| process.working_directory.text | Multi-field of `process.working_directory`. | match_only_text | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.description | The description of the rule generating the event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| rule.version | The version / revision of the rule being used for analysis. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| threat.enrichments | A list of associated indicators objects enriching the event, and the context of that association/enrichment. | nested | -| threat.framework | Name of the threat framework used to further categorize and classify the tactic and technique of the reported threat. Framework classification can be provided by detecting systems, evaluated at ingest time, or retrospectively tagged to events. | keyword | -| threat.indicator.file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| threat.indicator.file.path.text | Multi-field of `threat.indicator.file.path`. | match_only_text | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.software.platforms | The platforms of the software used by this threat to conduct behavior commonly modeled using MITRE ATT&CK®. While not required, you can use MITRE ATT&CK® software platform values. | keyword | -| threat.tactic.id | The id of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/ ) | keyword | -| threat.tactic.name | Name of the type of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/) | keyword | -| threat.tactic.reference | The reference url of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/ ) | keyword | -| threat.technique.id | The id of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.name | The name of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.name.text | Multi-field of `threat.technique.name`. | match_only_text | -| threat.technique.reference | The reference url of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.effective.id | Unique identifier of the user. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | | volume.bus_type | | keyword | | volume.device_name | | keyword | | volume.file_system_type | | keyword | @@ -979,157 +659,17 @@ An example event for `web_threat_events` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.image.tag | Container image tags. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| container.runtime | Runtime managing this container. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Name of the dataset. | constant_keyword | -| event.duration | Duration of the event in nanoseconds. If `event.start` and `event.end` are known this value should be the difference between the end and start time. | long | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.code_signature.signing_id | The identifier used to sign the process. This is used to identify the application manufactured by a software vendor. The field is relevant to Apple \*OS only. | keyword | -| file.code_signature.status | Additional information about the certificate status. This is useful for logging cryptographic errors with the certificate validity or trust status. Leave unpopulated if the validity or trust of the certificate was unchecked. | keyword | -| file.code_signature.team_id | The team identifier used to sign the process. This is used to identify the team or vendor of a software product. The field is relevant to Apple \*OS only. | keyword | -| file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| file.gid | Primary group ID (GID) of the file. | keyword | -| file.hash.md5 | MD5 hash. | keyword | -| file.hash.sha1 | SHA1 hash. | keyword | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.hash.sha512 | SHA512 hash. | keyword | -| file.inode | Inode representing the file in the filesystem. | keyword | -| file.mode | Mode of the file in octal representation. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| file.uid | The user ID (UID) or security identifier (SID) of the file owner. | keyword | -| group.id | Unique identifier for the group on the system/platform. | keyword | -| group.name | Name of the group. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.geo.country_iso_code | Country ISO code. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | -| log.logger | The name of the logger inside an application. This is usually the name of the class which initialized the logger, or can be a custom name. | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| observer.name | Custom name of the observer. This is a name that can be given to an observer. This can be helpful for example if multiple firewalls of the same model are used in an organization. If no custom name is needed, the field can be left empty. | keyword | -| observer.product | The product name of the observer. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| organization.id | | keyword | -| process.args | Array of process arguments, starting with the absolute path to the executable. May be filtered to protect sensitive information. | keyword | -| process.args_count | Length of the process.args array. This field can be useful for querying or performing bucket analysis on how many arguments were provided to start a process. More arguments may be an indication of suspicious activity. | long | -| process.code_signature.signing_id | The identifier used to sign the process. This is used to identify the application manufactured by a software vendor. The field is relevant to Apple \*OS only. | keyword | -| process.code_signature.status | Additional information about the certificate status. This is useful for logging cryptographic errors with the certificate validity or trust status. Leave unpopulated if the validity or trust of the certificate was unchecked. | keyword | -| process.code_signature.team_id | The team identifier used to sign the process. This is used to identify the team or vendor of a software product. The field is relevant to Apple \*OS only. | keyword | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.executable | Absolute path to the process executable. | keyword | -| process.executable.text | Multi-field of `process.executable`. | match_only_text | -| process.exit_code | The exit code of the process, if this is a termination event. The field should be absent if there is no exit code for the event (e.g. process start). | long | -| process.group_leader.pid | Process id. | long | -| process.hash.md5 | MD5 hash. | keyword | -| process.hash.sha1 | SHA1 hash. | keyword | -| process.hash.sha256 | SHA256 hash. | keyword | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.parent.pid | Process id. | long | -| process.parent.start | The time the process started. | date | -| process.pid | Process id. | long | -| process.real_group.id | Unique identifier for the group on the system/platform. | keyword | -| process.real_user.id | Unique identifier of the user. | keyword | -| process.start | The time the process started. | date | -| process.tty.char_device.major | The major number identifies the driver associated with the device. The character device's major and minor numbers can be algorithmically combined to produce the more familiar terminal identifiers such as "ttyS0" and "pts/0". For more details, please refer to the Linux kernel documentation. | long | -| process.tty.char_device.minor | The minor number is used only by the driver specified by the major number; other parts of the kernel don’t use it, and merely pass it along to the driver. It is common for a driver to control several devices; the minor number provides a way for the driver to differentiate among them. | long | -| process.tty.columns | The number of character columns per line. e.g terminal width Terminal sizes can change, so this value reflects the maximum value for a given IO event. i.e. where event.action = 'text_output' | long | -| process.tty.rows | The number of character rows in the terminal. e.g terminal height Terminal sizes can change, so this value reflects the maximum value for a given IO event. i.e. where event.action = 'text_output' | long | -| process.user.id | Unique identifier of the user. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.description | The description of the rule generating the event. | keyword | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| threat.enrichments | A list of associated indicators objects enriching the event, and the context of that association/enrichment. | nested | -| threat.framework | Name of the threat framework used to further categorize and classify the tactic and technique of the reported threat. Framework classification can be provided by detecting systems, evaluated at ingest time, or retrospectively tagged to events. | keyword | -| threat.software.platforms | The platforms of the software used by this threat to conduct behavior commonly modeled using MITRE ATT&CK®. While not required, you can use MITRE ATT&CK® software platform values. | keyword | -| threat.tactic.id | The id of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/ ) | keyword | -| threat.tactic.name | Name of the type of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/) | keyword | -| threat.tactic.reference | The reference url of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/ ) | keyword | -| threat.technique.id | The id of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.name | The name of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.name.text | Multi-field of `threat.technique.name`. | match_only_text | -| threat.technique.reference | The reference url of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | | volume.bus_type | | keyword | | volume.file_system_type | | keyword | | volume.nt_name | | keyword | @@ -1244,128 +784,15 @@ An example event for `web_traffic_events` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| dns.answers.ttl | The time interval in seconds that this resource record may be cached before it should be discarded. Zero values mean that the data should not be cached. | long | -| dns.answers.type | The type of data contained in this resource record. | keyword | -| dns.question.name | The name being queried. If the name field contains non-printable characters (below 32 or above 126), those characters should be represented as escaped base 10 integers (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, and line feeds should be converted to \t, \r, and \n respectively. | keyword | -| dns.question.registered_domain | The highest registered domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| dns.question.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| dns.response_code | The DNS response code. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Name of the dataset. | constant_keyword | -| event.duration | Duration of the event in nanoseconds. If `event.start` and `event.end` are known this value should be the difference between the end and start time. | long | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.code_signature.signing_id | The identifier used to sign the process. This is used to identify the application manufactured by a software vendor. The field is relevant to Apple \*OS only. | keyword | -| file.code_signature.status | Additional information about the certificate status. This is useful for logging cryptographic errors with the certificate validity or trust status. Leave unpopulated if the validity or trust of the certificate was unchecked. | keyword | -| file.code_signature.team_id | The team identifier used to sign the process. This is used to identify the team or vendor of a software product. The field is relevant to Apple \*OS only. | keyword | -| file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| file.gid | Primary group ID (GID) of the file. | keyword | -| file.hash.md5 | MD5 hash. | keyword | -| file.hash.sha1 | SHA1 hash. | keyword | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.hash.sha512 | SHA512 hash. | keyword | -| file.inode | Inode representing the file in the filesystem. | keyword | -| file.mode | Mode of the file in octal representation. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| file.uid | The user ID (UID) or security identifier (SID) of the file owner. | keyword | -| group.id | Unique identifier for the group on the system/platform. | keyword | -| group.name | Name of the group. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.geo.country_iso_code | Country ISO code. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | -| log.logger | The name of the logger inside an application. This is usually the name of the class which initialized the logger, or can be a custom name. | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| observer.name | Custom name of the observer. This is a name that can be given to an observer. This can be helpful for example if multiple firewalls of the same model are used in an organization. If no custom name is needed, the field can be left empty. | keyword | -| observer.product | The product name of the observer. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| organization.id | | keyword | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.parent.pid | Process id. | long | -| process.parent.start | The time the process started. | date | -| process.pid | Process id. | long | -| process.start | The time the process started. | date | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.description | The description of the rule generating the event. | keyword | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | diff --git a/packages/jamf_protect/manifest.yml b/packages/jamf_protect/manifest.yml index 57199646a62..9c7ad3b9eff 100644 --- a/packages/jamf_protect/manifest.yml +++ b/packages/jamf_protect/manifest.yml @@ -1,14 +1,14 @@ format_version: 3.0.3 name: jamf_protect title: Jamf Protect -version: "2.0.0" +version: "2.1.0" description: Receives events from Jamf Protect with Elastic Agent. type: integration categories: - security conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" screenshots: - src: /img/jamfprotect_kibana.png title: Jamf Protect Kibana From 2de821d412c54bbd57d4e4171b9e8f7d80922149 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:25:39 +0930 Subject: [PATCH 056/121] [jumpcloud] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/jumpcloud --- packages/jumpcloud/changelog.yml | 5 + .../data_stream/events/fields/ecs.yml | 122 ------------------ packages/jumpcloud/docs/README.md | 68 ---------- packages/jumpcloud/manifest.yml | 4 +- 4 files changed, 7 insertions(+), 192 deletions(-) delete mode 100644 packages/jumpcloud/data_stream/events/fields/ecs.yml diff --git a/packages/jumpcloud/changelog.yml b/packages/jumpcloud/changelog.yml index b203d88a642..9e0f151bbb5 100644 --- a/packages/jumpcloud/changelog.yml +++ b/packages/jumpcloud/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.11.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.10.0" changes: - description: Set sensitive values as secret. diff --git a/packages/jumpcloud/data_stream/events/fields/ecs.yml b/packages/jumpcloud/data_stream/events/fields/ecs.yml deleted file mode 100644 index bf1133de808..00000000000 --- a/packages/jumpcloud/data_stream/events/fields/ecs.yml +++ /dev/null @@ -1,122 +0,0 @@ -- external: ecs - name: client.ip -- external: ecs - name: client.as.number -- external: ecs - name: client.as.organization.name -- external: ecs - name: client.bytes -- external: ecs - name: client.domain -- external: ecs - name: client.geo.city_name -- external: ecs - name: client.geo.continent_name -- external: ecs - name: client.geo.country_iso_code -- external: ecs - name: client.geo.country_name -- external: ecs - name: client.geo.location -- external: ecs - name: client.geo.region_iso_code -- external: ecs - name: client.geo.region_name -- external: ecs - name: ecs.version -- external: ecs - name: event.ingested -- external: ecs - name: event.original -- external: ecs - name: message -- external: ecs - name: process.name -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: rule.category -- external: ecs - name: rule.id -- external: ecs - name: rule.uuid -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.bytes -- external: ecs - name: source.domain -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: source.mac -- external: ecs - name: source.nat.ip -- external: ecs - name: source.nat.port -- external: ecs - name: source.port -- external: ecs - name: source.user.email -- external: ecs - name: source.user.id -- external: ecs - name: source.user.name -- external: ecs - name: tags -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user.email -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version -- name: event.action - external: ecs -- name: event.category - external: ecs -- name: event.code - external: ecs -- name: event.duration - external: ecs -- name: event.id - external: ecs -- name: event.kind - external: ecs -- name: event.risk_score - external: ecs -- name: event.severity - external: ecs diff --git a/packages/jumpcloud/docs/README.md b/packages/jumpcloud/docs/README.md index a133de54f2e..53c3179cf18 100644 --- a/packages/jumpcloud/docs/README.md +++ b/packages/jumpcloud/docs/README.md @@ -48,33 +48,9 @@ All JumpCloud Directory Insights events are available in the `jumpcloud.events` | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| client.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| client.as.organization.name | Organization name. | keyword | -| client.as.organization.name.text | Multi-field of `client.as.organization.name`. | match_only_text | -| client.bytes | Bytes sent from the client to the server. | long | -| client.domain | The domain name of the client system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| client.geo.city_name | City name. | keyword | -| client.geo.continent_name | Name of the continent. | keyword | -| client.geo.country_iso_code | Country ISO code. | keyword | -| client.geo.country_name | Country name. | keyword | -| client.geo.location | Longitude and latitude. | geo_point | -| client.geo.region_iso_code | Region ISO code. | keyword | -| client.geo.region_name | Region name. | keyword | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | -| event.duration | Duration of the event in nanoseconds. If `event.start` and `event.end` are known this value should be the difference between the end and start time. | long | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.risk_score | Risk score or priority of the event (e.g. security solutions). Use your system's original value here. | float | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | | input.type | | keyword | | jumpcloud.event.application.display_label | | keyword | | jumpcloud.event.application.id | | keyword | @@ -181,50 +157,6 @@ All JumpCloud Directory Insights events are available in the `jumpcloud.events` | jumpcloud.event.useragent.version | | keyword | | jumpcloud.event.username | | keyword | | jumpcloud.event.version | | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.category | A categorization value keyword used by the entity using the rule for detection of this event. | keyword | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| rule.uuid | A rule ID that is unique within the scope of a set or group of agents, observers, or other entities using the rule for detection of this event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.mac | MAC address of the source. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| source.nat.ip | Translated ip of source based NAT sessions (e.g. internal client to internet) Typically connections traversing load balancers, firewalls, or routers. | ip | -| source.nat.port | Translated port of source based NAT sessions. (e.g. internal client to internet) Typically used with load balancers, firewalls, or routers. | long | -| source.port | Port of the source. | long | -| source.user.email | User email address. | keyword | -| source.user.id | Unique identifier of the user. | keyword | -| source.user.name | Short name or login of the user. | keyword | -| source.user.name.text | Multi-field of `source.user.name`. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | An example event for `events` looks as following: diff --git a/packages/jumpcloud/manifest.yml b/packages/jumpcloud/manifest.yml index 59820bdfa92..81ffd8fd3c6 100644 --- a/packages/jumpcloud/manifest.yml +++ b/packages/jumpcloud/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: jumpcloud title: "JumpCloud" -version: "1.10.0" +version: "1.11.0" description: "Collect logs from JumpCloud Directory as a Service" type: integration categories: @@ -9,7 +9,7 @@ categories: - security conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" elastic: subscription: "basic" screenshots: From e0c703dec6f0ed2f381682c72757e0bf3d16c4b2 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:25:40 +0930 Subject: [PATCH 057/121] [keycloak] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^7.16.0 || ^8.0.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/keycloak --- packages/keycloak/changelog.yml | 5 + .../keycloak/data_stream/log/fields/agent.yml | 167 +----------------- .../keycloak/data_stream/log/fields/beats.yml | 3 - .../keycloak/data_stream/log/fields/ecs.yml | 86 --------- .../keycloak/data_stream/log/manifest.yml | 2 +- packages/keycloak/docs/README.md | 73 -------- packages/keycloak/manifest.yml | 4 +- 7 files changed, 9 insertions(+), 331 deletions(-) diff --git a/packages/keycloak/changelog.yml b/packages/keycloak/changelog.yml index 73d4d82daa8..b2def31fcf5 100644 --- a/packages/keycloak/changelog.yml +++ b/packages/keycloak/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.22.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.21.0" changes: - description: Update manifest format version to v3.0.3. diff --git a/packages/keycloak/data_stream/log/fields/agent.yml b/packages/keycloak/data_stream/log/fields/agent.yml index da4e652c53b..2bc58530bac 100644 --- a/packages/keycloak/data_stream/log/fields/agent.yml +++ b/packages/keycloak/data_stream/log/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/keycloak/data_stream/log/fields/beats.yml b/packages/keycloak/data_stream/log/fields/beats.yml index 4e189f20187..b2c7e0a2961 100644 --- a/packages/keycloak/data_stream/log/fields/beats.yml +++ b/packages/keycloak/data_stream/log/fields/beats.yml @@ -7,9 +7,6 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. - name: log.file type: group fields: diff --git a/packages/keycloak/data_stream/log/fields/ecs.yml b/packages/keycloak/data_stream/log/fields/ecs.yml index 6d3428b27d2..7a327dae36b 100644 --- a/packages/keycloak/data_stream/log/fields/ecs.yml +++ b/packages/keycloak/data_stream/log/fields/ecs.yml @@ -1,87 +1 @@ -- name: ecs.version - external: ecs -- name: error.message - external: ecs -- name: event.action - external: ecs -- name: event.category - external: ecs -- name: event.id - external: ecs -- name: event.ingested - external: ecs -- name: event.created - external: ecs -- name: event.kind - external: ecs -- name: event.original - external: ecs -- name: event.type - external: ecs -- name: message - external: ecs -- name: related.ip - external: ecs -- name: related.user - external: ecs -- name: related.hosts - external: ecs -- name: source.address - external: ecs -- name: source.as.number - external: ecs -- name: source.as.organization.name - external: ecs -- name: source.bytes - external: ecs -- name: source.domain - external: ecs -- name: source.geo.city_name - external: ecs -- name: source.geo.continent_name - external: ecs -- name: source.geo.country_iso_code - external: ecs -- name: source.geo.country_name - external: ecs -- name: source.geo.location - external: ecs -- name: source.geo.name - external: ecs -- name: source.geo.region_iso_code - external: ecs -- name: source.geo.region_name - external: ecs -- name: source.ip - external: ecs -- name: source.port - external: ecs -- name: tags - external: ecs -- name: user.id - external: ecs -- name: user.name - external: ecs -- name: log.level - external: ecs -- name: log.logger - external: ecs -- name: process.thread.name - external: ecs -- name: group.id - external: ecs -- name: user.target.id - external: ecs -- name: url.domain - external: ecs -- name: url.extension - external: ecs -- name: url.fragment - external: ecs -- name: url.original - external: ecs -- name: url.path - external: ecs -- name: url.port - external: ecs - name: url.scheme diff --git a/packages/keycloak/data_stream/log/manifest.yml b/packages/keycloak/data_stream/log/manifest.yml index 9741ae8fb5c..30de5132a78 100644 --- a/packages/keycloak/data_stream/log/manifest.yml +++ b/packages/keycloak/data_stream/log/manifest.yml @@ -52,6 +52,7 @@ streams: show_user: false description: > Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: parsers type: yaml title: Parsers @@ -65,7 +66,6 @@ streams: pattern: '^\d{4}-\w{2}-\d{2}' negate: true match: after - template_path: "filestream.yml.hbs" title: Keycloak logs description: Collect Keycloak logs via log files diff --git a/packages/keycloak/docs/README.md b/packages/keycloak/docs/README.md index 31df48cd0e0..226db3908c1 100644 --- a/packages/keycloak/docs/README.md +++ b/packages/keycloak/docs/README.md @@ -31,52 +31,15 @@ Note: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| group.id | Unique identifier for the group on the system/platform. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | | keycloak.admin.operation | Keycloak admin operation; Add, Update, Delete | keyword | | keycloak.admin.resource.path | Path to affected resource | keyword | @@ -96,46 +59,10 @@ Note: | log.file.idxhi | The high-order part of a unique identifier that is associated with a file. (Windows-only) | keyword | | log.file.idxlo | The low-order part of a unique identifier that is associated with a file. (Windows-only) | keyword | | log.file.inode | Inode number of the log file. | keyword | -| log.file.path | Path to the log file. | keyword | | log.file.vol | The serial number of the volume that contains a file. (Windows-only) | keyword | | log.flags | Flags for the log file. | keyword | -| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | -| log.logger | The name of the logger inside an application. This is usually the name of the class which initialized the logger, or can be a custom name. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| process.thread.name | Thread name. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | | url.scheme | | | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user.target.id | Unique identifier of the user. | keyword | An example event for `log` looks as following: diff --git a/packages/keycloak/manifest.yml b/packages/keycloak/manifest.yml index 4970508ce32..f6452c0efe0 100644 --- a/packages/keycloak/manifest.yml +++ b/packages/keycloak/manifest.yml @@ -1,13 +1,13 @@ name: keycloak title: Keycloak -version: "1.21.0" +version: "1.22.0" description: Collect logs from Keycloak with Elastic Agent. type: integration format_version: "3.0.3" categories: [security, iam] conditions: kibana: - version: "^7.16.0 || ^8.0.0" + version: "^8.13.0" icons: - src: /img/keycloak-logo.svg title: Keycloak From 026d900e82ab54fcfedeea4234eec8d11bde4c56 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:25:43 +0930 Subject: [PATCH 058/121] [lastpass] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/lastpass --- packages/lastpass/changelog.yml | 5 + .../detailed_shared_folder/fields/agent.yml | 147 ------------------ .../detailed_shared_folder/fields/ecs.yml | 16 -- .../data_stream/event_report/fields/agent.yml | 147 ------------------ .../data_stream/event_report/fields/ecs.yml | 34 ---- .../data_stream/user/fields/agent.yml | 147 ------------------ .../lastpass/data_stream/user/fields/ecs.yml | 24 --- packages/lastpass/docs/README.md | 116 -------------- packages/lastpass/manifest.yml | 4 +- 9 files changed, 7 insertions(+), 633 deletions(-) delete mode 100644 packages/lastpass/data_stream/detailed_shared_folder/fields/ecs.yml delete mode 100644 packages/lastpass/data_stream/event_report/fields/ecs.yml delete mode 100644 packages/lastpass/data_stream/user/fields/ecs.yml diff --git a/packages/lastpass/changelog.yml b/packages/lastpass/changelog.yml index bded279947c..6966b838cf4 100644 --- a/packages/lastpass/changelog.yml +++ b/packages/lastpass/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.17.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.16.0" changes: - description: Add pagesize and pageindex to request. diff --git a/packages/lastpass/data_stream/detailed_shared_folder/fields/agent.yml b/packages/lastpass/data_stream/detailed_shared_folder/fields/agent.yml index 73e076a93b1..894e6f12be2 100644 --- a/packages/lastpass/data_stream/detailed_shared_folder/fields/agent.yml +++ b/packages/lastpass/data_stream/detailed_shared_folder/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/lastpass/data_stream/detailed_shared_folder/fields/ecs.yml b/packages/lastpass/data_stream/detailed_shared_folder/fields/ecs.yml deleted file mode 100644 index 16255512a07..00000000000 --- a/packages/lastpass/data_stream/detailed_shared_folder/fields/ecs.yml +++ /dev/null @@ -1,16 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: related.user -- external: ecs - name: tags -- external: ecs - name: user.email diff --git a/packages/lastpass/data_stream/event_report/fields/agent.yml b/packages/lastpass/data_stream/event_report/fields/agent.yml index 6e1bac042bc..894e6f12be2 100644 --- a/packages/lastpass/data_stream/event_report/fields/agent.yml +++ b/packages/lastpass/data_stream/event_report/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/lastpass/data_stream/event_report/fields/ecs.yml b/packages/lastpass/data_stream/event_report/fields/ecs.yml deleted file mode 100644 index 52216f81053..00000000000 --- a/packages/lastpass/data_stream/event_report/fields/ecs.yml +++ /dev/null @@ -1,34 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.ip -- external: ecs - name: tags -- external: ecs - name: user.email -- external: ecs - name: user.group.name diff --git a/packages/lastpass/data_stream/user/fields/agent.yml b/packages/lastpass/data_stream/user/fields/agent.yml index 6e1bac042bc..894e6f12be2 100644 --- a/packages/lastpass/data_stream/user/fields/agent.yml +++ b/packages/lastpass/data_stream/user/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/lastpass/data_stream/user/fields/ecs.yml b/packages/lastpass/data_stream/user/fields/ecs.yml deleted file mode 100644 index 97efa126a91..00000000000 --- a/packages/lastpass/data_stream/user/fields/ecs.yml +++ /dev/null @@ -1,24 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: related.user -- external: ecs - name: tags -- external: ecs - name: user.email -- external: ecs - name: user.full_name -- external: ecs - name: user.group.name -- external: ecs - name: user.id diff --git a/packages/lastpass/docs/README.md b/packages/lastpass/docs/README.md index ee51c087e60..ab3dc0415cf 100644 --- a/packages/lastpass/docs/README.md +++ b/packages/lastpass/docs/README.md @@ -128,46 +128,15 @@ An example event for `detailed_shared_folder` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | lastpass.detailed_shared_folder.deleted | | boolean | | lastpass.detailed_shared_folder.name | | keyword | @@ -180,9 +149,6 @@ An example event for `detailed_shared_folder` looks as following: | lastpass.detailed_shared_folder.user.site | | keyword | | lastpass.detailed_shared_folder.user.super_admin | | boolean | | log.offset | Log offset | long | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| user.email | User email address. | keyword | ### event_report @@ -275,48 +241,15 @@ An example event for `event_report` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | lastpass.event_report.action | | keyword | | lastpass.event_report.data.added_site | | keyword | @@ -338,16 +271,6 @@ An example event for `event_report` looks as following: | lastpass.event_report.time | | date | | lastpass.event_report.user_name | | keyword | | log.offset | Log offset | long | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| user.email | User email address. | keyword | -| user.group.name | Name of the group. | keyword | ### user @@ -456,47 +379,15 @@ An example event for `user` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | lastpass.user.application | | long | | lastpass.user.attachment | | long | @@ -518,11 +409,4 @@ An example event for `user` looks as following: | lastpass.user.total_score | | double | | lastpass.user.user_name | | keyword | | log.offset | Log offset | long | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| user.email | User email address. | keyword | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.group.name | Name of the group. | keyword | -| user.id | Unique identifier of the user. | keyword | diff --git a/packages/lastpass/manifest.yml b/packages/lastpass/manifest.yml index 15f7d40d175..f9a9de1d5d4 100644 --- a/packages/lastpass/manifest.yml +++ b/packages/lastpass/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: lastpass title: LastPass -version: "1.16.0" +version: "1.17.0" description: Collect logs from LastPass with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - credential_management conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: subscription: basic screenshots: From bcc3812c5e40253f1205d5b57d916e68d8655c00 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:25:44 +0930 Subject: [PATCH 059/121] [lumos] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.1 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/lumos --- packages/lumos/changelog.yml | 5 +++++ .../lumos/data_stream/activity_logs/fields/ecs.yml | 4 ---- .../data_stream/activity_logs/fields/fields.yml | 12 ------------ packages/lumos/docs/README.md | 6 ------ packages/lumos/manifest.yml | 4 ++-- 5 files changed, 7 insertions(+), 24 deletions(-) delete mode 100644 packages/lumos/data_stream/activity_logs/fields/ecs.yml diff --git a/packages/lumos/changelog.yml b/packages/lumos/changelog.yml index 98bad0e736a..b88153ed648 100644 --- a/packages/lumos/changelog.yml +++ b/packages/lumos/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.3.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.2.1" changes: - description: Fix sample event. diff --git a/packages/lumos/data_stream/activity_logs/fields/ecs.yml b/packages/lumos/data_stream/activity_logs/fields/ecs.yml deleted file mode 100644 index 553d3da3148..00000000000 --- a/packages/lumos/data_stream/activity_logs/fields/ecs.yml +++ /dev/null @@ -1,4 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message diff --git a/packages/lumos/data_stream/activity_logs/fields/fields.yml b/packages/lumos/data_stream/activity_logs/fields/fields.yml index 763d709168a..3b850446fd1 100644 --- a/packages/lumos/data_stream/activity_logs/fields/fields.yml +++ b/packages/lumos/data_stream/activity_logs/fields/fields.yml @@ -1,18 +1,6 @@ - name: input.type type: keyword description: Input type -- name: event.id - type: keyword - description: The event hash -- name: event.created - type: date - description: The time the event began -- name: event.action - type: keyword - description: The activity that occurred -- name: event.outcome - type: keyword - description: The outcome of the event, whether it succeeded or failed - name: lumos.activity_logs.actor.actor_type type: keyword description: The type of actor diff --git a/packages/lumos/docs/README.md b/packages/lumos/docs/README.md index 1becdd681e4..47db95c2def 100644 --- a/packages/lumos/docs/README.md +++ b/packages/lumos/docs/README.md @@ -34,12 +34,7 @@ Activity Logs summarize the history of changes and events occurring within Lumos | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The activity that occurred | keyword | -| event.created | The time the event began | date | -| event.id | The event hash | keyword | | event.module | Event module | constant_keyword | -| event.outcome | The outcome of the event, whether it succeeded or failed | keyword | | input.type | Input type | keyword | | lumos.activity_logs.actor.actor_type | The type of actor | keyword | | lumos.activity_logs.actor.email | The email of the actor | keyword | @@ -49,7 +44,6 @@ Activity Logs summarize the history of changes and events occurring within Lumos | lumos.activity_logs.event_type_user_friendly | The user friendly type of the event | keyword | | lumos.activity_logs.targets.name | | keyword | | lumos.activity_logs.targets.target_type | | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | An example event for `activity` looks as following: diff --git a/packages/lumos/manifest.yml b/packages/lumos/manifest.yml index 8d03976ac00..3984bed287c 100644 --- a/packages/lumos/manifest.yml +++ b/packages/lumos/manifest.yml @@ -1,14 +1,14 @@ format_version: 3.1.2 name: lumos title: "Lumos" -version: 1.2.1 +version: "1.3.0" description: "An integration with Lumos to ship your Activity logs to your Elastic instance." type: integration categories: - security conditions: kibana: - version: "^8.12.1" + version: "^8.13.0" elastic: subscription: "basic" screenshots: From f2ca79b52bf290c7a578e4c39fe8964ec79e8aa6 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:25:45 +0930 Subject: [PATCH 060/121] [lyve_cloud] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/lyve_cloud --- packages/lyve_cloud/changelog.yml | 5 + .../data_stream/audit/fields/agent.yml | 167 +----------------- .../data_stream/audit/fields/ecs.yml | 76 -------- packages/lyve_cloud/docs/README.md | 70 -------- packages/lyve_cloud/manifest.yml | 4 +- 5 files changed, 8 insertions(+), 314 deletions(-) diff --git a/packages/lyve_cloud/changelog.yml b/packages/lyve_cloud/changelog.yml index 272e7d0bfb4..ee84f4bdf41 100644 --- a/packages/lyve_cloud/changelog.yml +++ b/packages/lyve_cloud/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.14.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.13.0" changes: - description: Set sensitive values as secret. diff --git a/packages/lyve_cloud/data_stream/audit/fields/agent.yml b/packages/lyve_cloud/data_stream/audit/fields/agent.yml index da4e652c53b..2bc58530bac 100644 --- a/packages/lyve_cloud/data_stream/audit/fields/agent.yml +++ b/packages/lyve_cloud/data_stream/audit/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/lyve_cloud/data_stream/audit/fields/ecs.yml b/packages/lyve_cloud/data_stream/audit/fields/ecs.yml index 7136e8e2fda..2699ba80122 100644 --- a/packages/lyve_cloud/data_stream/audit/fields/ecs.yml +++ b/packages/lyve_cloud/data_stream/audit/fields/ecs.yml @@ -1,86 +1,10 @@ - external: ecs name: "@timestamp" -- external: ecs - name: ecs.version -- external: ecs - name: tags -- external: ecs - name: user.name -- external: ecs - name: user.email -- external: ecs - name: user.id -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: http.response.status_code -- external: ecs - name: http.request.body.bytes -- external: ecs - name: http.response.body.bytes -- external: ecs - name: http.response.mime_type -- external: ecs - name: log.file.path -- external: ecs - name: user_agent.version -- external: ecs - name: related.user -- external: ecs - name: related.ip -- external: ecs - name: related.hosts -- external: ecs - name: client.ip -- external: ecs - name: client.geo.continent_name -- external: ecs - name: client.geo.city_name -- external: ecs - name: client.geo.country_iso_code -- external: ecs - name: client.geo.country_name - external: ecs name: client.geo.location.lat - external: ecs name: client.geo.location.lon -- external: ecs - name: client.geo.region_iso_code -- external: ecs - name: client.geo.region_name -- external: ecs - name: client.as.number -- external: ecs - name: client.as.organization.name -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.ip -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name - external: ecs name: source.geo.location.lat - external: ecs name: source.geo.location.lon -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name diff --git a/packages/lyve_cloud/docs/README.md b/packages/lyve_cloud/docs/README.md index 3ea91d5e412..20994962bbf 100644 --- a/packages/lyve_cloud/docs/README.md +++ b/packages/lyve_cloud/docs/README.md @@ -29,60 +29,18 @@ when creating new dashboard or in other Analytics search fields inside the filte | Field | Description | Type | |---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | -| client.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| client.as.organization.name | Organization name. | keyword | -| client.as.organization.name.text | Multi-field of `client.as.organization.name`. | match_only_text | -| client.geo.city_name | City name. | keyword | -| client.geo.continent_name | Name of the continent. | keyword | -| client.geo.country_iso_code | Country ISO code. | keyword | -| client.geo.country_name | Country name. | keyword | | client.geo.location.lat | Longitude and latitude. | geo_point | | client.geo.location.lon | Longitude and latitude. | geo_point | -| client.geo.region_iso_code | Region ISO code. | keyword | -| client.geo.region_name | Region name. | keyword | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | event.dataset | Event dataset | constant_keyword | | event.module | Event module | constant_keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.body.bytes | Size in bytes of the request body. | long | -| http.response.body.bytes | Size in bytes of the response body. | long | -| http.response.mime_type | Mime type of the body of the response. This value must only be populated based on the content of the response body, not on the `Content-Type` header. Comparing the mime type of a response with the response's Content-Type header can be helpful in detecting misconfigured servers. | keyword | -| http.response.status_code | HTTP response status code. | long | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | lyve_cloud.audit.auditEntry.api.bucket | Bucket for which the opearion was taken upon. | keyword | | lyve_cloud.audit.auditEntry.api.name | Represents name of the operation. | keyword | | lyve_cloud.audit.auditEntry.api.object | Objects name | keyword | @@ -101,36 +59,8 @@ when creating new dashboard or in other Analytics search fields inside the filte | lyve_cloud.audit.auditEntry.responseHeader.object_lock_retain_until_date | Object retention duration | date | | lyve_cloud.audit.auditEntry.responseHeader.x-amz-version-id | The version of the object. When versioning is enabled. | keyword | | lyve_cloud.audit.auditEntry.version | Represents the current version of Audit Log structure. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | | source.geo.location.lat | Longitude and latitude. | geo_point | | source.geo.location.lon | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | An example event for `audit` looks as following: diff --git a/packages/lyve_cloud/manifest.yml b/packages/lyve_cloud/manifest.yml index 1796b2f6000..422be489dc9 100644 --- a/packages/lyve_cloud/manifest.yml +++ b/packages/lyve_cloud/manifest.yml @@ -1,14 +1,14 @@ format_version: "3.0.2" name: lyve_cloud title: Lyve Cloud -version: "1.13.0" +version: "1.14.0" description: Collect S3 API audit log from Lyve Cloud with Elastic Agent. type: integration categories: - security conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" icons: - src: /img/LyveCloud-Logo.svg title: Seagate-Lyve-Cloud From 3700f8e6cc0f31993643e2b90499532c464ebadf Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:25:52 +0930 Subject: [PATCH 061/121] [m365_defender] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/m365_defender --- packages/m365_defender/_dev/build/build.yml | 1 - packages/m365_defender/changelog.yml | 5 + .../data_stream/alert/fields/beats.yml | 3 - .../pipeline/test-device.log-expected.json | 2 + .../data_stream/event/fields/agent.yml | 147 -------- .../data_stream/event/fields/ecs.yml | 306 --------------- .../data_stream/incident/fields/agent.yml | 147 -------- .../data_stream/incident/fields/ecs.yml | 104 ----- .../data_stream/log/fields/agent.yml | 167 +------- .../data_stream/log/fields/ecs.yml | 90 ----- packages/m365_defender/docs/README.md | 356 ------------------ packages/m365_defender/manifest.yml | 4 +- 12 files changed, 10 insertions(+), 1322 deletions(-) delete mode 100644 packages/m365_defender/data_stream/incident/fields/ecs.yml delete mode 100644 packages/m365_defender/data_stream/log/fields/ecs.yml diff --git a/packages/m365_defender/_dev/build/build.yml b/packages/m365_defender/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/m365_defender/_dev/build/build.yml +++ b/packages/m365_defender/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/m365_defender/changelog.yml b/packages/m365_defender/changelog.yml index 3529064c4fa..496046b12fc 100644 --- a/packages/m365_defender/changelog.yml +++ b/packages/m365_defender/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.13.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.12.0" changes: - description: Make `host.ip` and `host.mac` fields conform to ECS field definition. diff --git a/packages/m365_defender/data_stream/alert/fields/beats.yml b/packages/m365_defender/data_stream/alert/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/m365_defender/data_stream/alert/fields/beats.yml +++ b/packages/m365_defender/data_stream/alert/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/m365_defender/data_stream/event/_dev/test/pipeline/test-device.log-expected.json b/packages/m365_defender/data_stream/event/_dev/test/pipeline/test-device.log-expected.json index b113b2a5bae..85afb6f8534 100644 --- a/packages/m365_defender/data_stream/event/_dev/test/pipeline/test-device.log-expected.json +++ b/packages/m365_defender/data_stream/event/_dev/test/pipeline/test-device.log-expected.json @@ -2020,6 +2020,7 @@ "preserve_duplicate_custom_fields" ], "url": { + "extension": "tld", "original": "subdomain.domain.tld", "path": "subdomain.domain.tld" }, @@ -2718,6 +2719,7 @@ "preserve_duplicate_custom_fields" ], "url": { + "extension": "com", "original": "url.com", "path": "url.com" }, diff --git a/packages/m365_defender/data_stream/event/fields/agent.yml b/packages/m365_defender/data_stream/event/fields/agent.yml index 6e1bac042bc..894e6f12be2 100644 --- a/packages/m365_defender/data_stream/event/fields/agent.yml +++ b/packages/m365_defender/data_stream/event/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/m365_defender/data_stream/event/fields/ecs.yml b/packages/m365_defender/data_stream/event/fields/ecs.yml index 9b0fe006804..c4589bf1b4b 100644 --- a/packages/m365_defender/data_stream/event/fields/ecs.yml +++ b/packages/m365_defender/data_stream/event/fields/ecs.yml @@ -1,308 +1,2 @@ -- external: ecs - name: ecs.version -- external: ecs - name: tags -- external: ecs - name: message -- external: ecs - name: destination.address -- external: ecs - name: destination.domain -- external: ecs - name: destination.ip -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.port -- external: ecs - name: dns.header_flags -- external: ecs - name: dns.question.class -- external: ecs - name: dns.question.name -- external: ecs - name: dns.question.type -- external: ecs - name: dns.response_code -- external: ecs - name: email.direction -- external: ecs - name: email.from.address -- external: ecs - name: email.local_id -- external: ecs - name: email.message_id -- external: ecs - name: email.subject -- external: ecs - name: email.to.address -- external: ecs - name: event.action -- external: ecs - name: event.outcome -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.original -- external: ecs - name: event.provider -- external: ecs - name: event.reference -- external: ecs - name: event.severity -- external: ecs - name: event.category -- external: ecs - name: event.type -- external: ecs - name: event.kind -- external: ecs - name: file.directory -- external: ecs - name: file.path -- external: ecs - name: file.extension -- external: ecs - name: file.hash.md5 -- external: ecs - name: file.hash.sha1 -- external: ecs - name: file.hash.sha256 -- external: ecs - name: file.name -- external: ecs - name: file.size -- external: ecs - name: file.x509.not_after -- external: ecs - name: file.x509.serial_number -- external: ecs - name: file.x509.issuer.common_name -- external: ecs - name: file.code_signature.subject_name -- external: ecs - name: file.code_signature.exists -- external: ecs - name: file.code_signature.trusted -- external: ecs - name: dll.path -- external: ecs - name: dll.name -- external: ecs - name: dll.hash.md5 -- external: ecs - name: dll.hash.sha1 -- external: ecs - name: dll.hash.sha256 -- external: ecs - name: network.direction -- external: ecs - name: network.protocol -- external: ecs - name: observer.type -- external: ecs - name: observer.version -- external: ecs - name: process.command_line -- external: ecs - name: process.start -- external: ecs - name: process.args -- external: ecs - name: process.args_count -- external: ecs - name: process.hash.md5 -- external: ecs - name: process.hash.sha1 -- external: ecs - name: process.hash.sha256 -- external: ecs - name: process.pid -- external: ecs - name: process.executable -- external: ecs - name: process.name -- external: ecs - name: process.pe.company -- external: ecs - name: process.pe.description -- external: ecs - name: process.pe.original_file_name -- external: ecs - name: process.pe.product -- external: ecs - name: process.pe.file_version -- external: ecs - name: process.code_signature.status -- external: ecs - name: process.parent.pid -- external: ecs - name: process.parent.start -- external: ecs - name: process.parent.command_line -- external: ecs - name: process.parent.args -- external: ecs - name: process.parent.args_count -- external: ecs - name: process.parent.hash.md5 -- external: ecs - name: process.parent.hash.sha1 -- external: ecs - name: process.parent.hash.sha256 -- external: ecs - name: process.parent.executable -- external: ecs - name: process.parent.name -- external: ecs - name: process.parent.pe.company -- external: ecs - name: process.parent.pe.description -- external: ecs - name: process.parent.pe.original_file_name -- external: ecs - name: process.parent.pe.product -- external: ecs - name: process.parent.pe.file_version -- external: ecs - name: process.parent.code_signature.status -- external: ecs - name: process.parent.code_signature.exists -- external: ecs - name: process.parent.code_signature.trusted -- external: ecs - name: process.parent.group_leader.pid -- external: ecs - name: process.parent.group_leader.start -# Missing in ECS flatfile - name: process.parent.group_leader.name type: keyword -- name: dns.answers - type: object - object_type: keyword -- external: ecs - name: registry.key -- external: ecs - name: registry.value -- external: ecs - name: registry.hive -- external: ecs - name: registry.path -- external: ecs - name: registry.data.strings -- external: ecs - name: registry.data.type -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: host.os.full -- external: ecs - name: user.domain -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: source.domain -- external: ecs - name: source.user.domain -- external: ecs - name: source.user.id -- external: ecs - name: source.user.name -- external: ecs - name: threat.indicator.file.directory -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.file.name -- external: ecs - name: threat.indicator.file.size -- external: ecs - name: threat.indicator.registry.key -- external: ecs - name: threat.indicator.registry.data.strings -- external: ecs - name: threat.indicator.registry.value -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.group.name -- external: ecs - name: threat.technique.subtechnique.id -- external: ecs - name: threat.technique.subtechnique.name -- external: ecs - name: url.domain -- external: ecs - name: url.extension -- external: ecs - name: url.fragment -- external: ecs - name: url.original -- external: ecs - name: url.password -- external: ecs - name: url.path -- external: ecs - name: url.port -- external: ecs - name: url.query -- external: ecs - name: url.scheme -- external: ecs - name: url.username -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version diff --git a/packages/m365_defender/data_stream/incident/fields/agent.yml b/packages/m365_defender/data_stream/incident/fields/agent.yml index 6e1bac042bc..894e6f12be2 100644 --- a/packages/m365_defender/data_stream/incident/fields/agent.yml +++ b/packages/m365_defender/data_stream/incident/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/m365_defender/data_stream/incident/fields/ecs.yml b/packages/m365_defender/data_stream/incident/fields/ecs.yml deleted file mode 100644 index 1652e216e7e..00000000000 --- a/packages/m365_defender/data_stream/incident/fields/ecs.yml +++ /dev/null @@ -1,104 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: email.delivery_timestamp -- external: ecs - name: email.direction -- external: ecs - name: email.from.address -- external: ecs - name: email.subject -- external: ecs - name: email.to.address -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.provider -- external: ecs - name: event.severity -- external: ecs - name: event.type -- external: ecs - name: event.url -- external: ecs - name: file.hash.sha1 -- external: ecs - name: file.hash.sha256 -- external: ecs - name: file.name -- external: ecs - name: file.path -- external: ecs - name: file.size -- external: ecs - name: group.id -- external: ecs - name: group.name -- external: ecs - name: message -- external: ecs - name: process.command_line -- external: ecs - name: process.hash.sha1 -- external: ecs - name: process.hash.sha256 -- external: ecs - name: process.parent.hash.sha1 -- external: ecs - name: process.parent.hash.sha256 -- external: ecs - name: process.parent.pid -- external: ecs - name: process.parent.start -- external: ecs - name: process.pid -- external: ecs - name: process.start -- external: ecs - name: process.user.id -- external: ecs - name: process.user.name -- external: ecs - name: registry.data.type -- external: ecs - name: registry.hive -- external: ecs - name: registry.key -- external: ecs - name: registry.value -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: source.ip -- external: ecs - name: source.user.name -- external: ecs - name: tags -- external: ecs - name: threat.group.name -- external: ecs - name: threat.tactic.name -- external: ecs - name: threat.technique.subtechnique.id -- external: ecs - name: user.domain -- external: ecs - name: user.id -- external: ecs - name: user.name diff --git a/packages/m365_defender/data_stream/log/fields/agent.yml b/packages/m365_defender/data_stream/log/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/m365_defender/data_stream/log/fields/agent.yml +++ b/packages/m365_defender/data_stream/log/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/m365_defender/data_stream/log/fields/ecs.yml b/packages/m365_defender/data_stream/log/fields/ecs.yml deleted file mode 100644 index 103c84ac3c7..00000000000 --- a/packages/m365_defender/data_stream/log/fields/ecs.yml +++ /dev/null @@ -1,90 +0,0 @@ -- external: ecs - name: event.kind -- external: ecs - name: event.timezone -- external: ecs - name: event.action -- external: ecs - name: event.provider -- external: ecs - name: event.created -- external: ecs - name: event.category -- external: ecs - name: event.type -- external: ecs - name: error.message -- external: ecs - name: event.id -- external: ecs - name: event.start -- external: ecs - name: event.end -- external: ecs - name: event.severity -- external: ecs - name: threat.framework -- external: ecs - name: threat.technique.name -- external: ecs - name: rule.description -- external: ecs - name: file.name -- external: ecs - name: file.hash.sha256 -- external: ecs - name: file.hash.sha1 -- external: ecs - name: file.path -- external: ecs - name: process.pid -- external: ecs - name: process.command_line -- external: ecs - name: process.start -- external: ecs - name: process.parent.pid -- external: ecs - name: process.parent.start -- external: ecs - name: observer.product -- external: ecs - name: observer.vendor -- external: ecs - name: observer.name -- external: ecs - name: url.domain -- external: ecs - name: url.full -- external: ecs - name: url.extension -- external: ecs - name: url.original -- external: ecs - name: url.path -- external: ecs - name: url.port -- external: ecs - name: url.scheme -- external: ecs - name: url.query -- external: ecs - name: user.name -- external: ecs - name: user.domain -- external: ecs - name: user.id -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: tags -- external: ecs - name: ecs.version -- external: ecs - name: message diff --git a/packages/m365_defender/docs/README.md b/packages/m365_defender/docs/README.md index 6aa34122da6..fcd0d8db391 100644 --- a/packages/m365_defender/docs/README.md +++ b/packages/m365_defender/docs/README.md @@ -553,7 +553,6 @@ An example event for `alert` looks as following: | m365_defender.alert.web_url.query | | keyword | | m365_defender.alert.web_url.scheme | | keyword | | m365_defender.alert.web_url.username | | keyword | -| tags | User defined tags. | keyword | ### event @@ -573,100 +572,16 @@ This is the `event` dataset. | Target.process.executable.text | Multi-field of `Target.process.executable`. | text | | Target.process.name | Process name. Sometimes called program name or similar. | keyword | | Target.process.name.text | Multi-field of `Target.process.name`. | text | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | | dll.Ext.size | Size of the dll executable. | long | -| dll.hash.md5 | MD5 hash. | keyword | -| dll.hash.sha1 | SHA1 hash. | keyword | -| dll.hash.sha256 | SHA256 hash. | keyword | -| dll.name | Name of the library. This generally maps to the name of the file on disk. | keyword | -| dll.path | Full file path of the library. | keyword | -| dns.answers | | object | -| dns.header_flags | Array of 2 letter DNS header flags. | keyword | -| dns.question.class | The class of records being queried. | keyword | -| dns.question.name | The name being queried. If the name field contains non-printable characters (below 32 or above 126), those characters should be represented as escaped base 10 integers (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, and line feeds should be converted to \t, \r, and \n respectively. | keyword | -| dns.question.type | The type of record being queried. | keyword | -| dns.response_code | The DNS response code. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.direction | The direction of the message based on the sending and receiving domains. | keyword | -| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | -| email.local_id | Unique identifier given to the email by the source that created the event. Identifier is not persistent across hops. | keyword | -| email.message_id | Identifier from the RFC 5322 `Message-ID:` email header that refers to a particular email message. | wildcard | -| email.subject | A brief summary of the topic of the message. | keyword | -| email.subject.text | Multi-field of `email.subject`. | match_only_text | -| email.to.address | The email address of recipient | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.reference | Reference URL linking to additional information about this event. This URL links to a static definition of this event. Alert events, indicated by `event.kind:alert`, are a common use case for this field. | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.code_signature.exists | Boolean to capture if a signature is present. | boolean | -| file.code_signature.subject_name | Subject name of the code signer | keyword | -| file.code_signature.trusted | Stores the trust status of the certificate chain. Validating the trust of the certificate chain may be complicated, and this field should only be populated by tools that actively check the status. | boolean | -| file.directory | Directory where the file is located. It should include the drive letter, when appropriate. | keyword | -| file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| file.hash.md5 | MD5 hash. | keyword | -| file.hash.sha1 | SHA1 hash. | keyword | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| file.x509.issuer.common_name | List of common name (CN) of issuing certificate authority. | keyword | -| file.x509.not_after | Time at which the certificate is no longer considered valid. | date | -| file.x509.serial_number | Unique serial number issued by the certificate authority. For consistency, if this value is alphanumeric, it should be formatted without colons and uppercase characters. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.full | Operating system name, including the version or code name. | keyword | -| host.os.full.text | Multi-field of `host.os.full`. | match_only_text | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | m365_defender.event.aad_device_id | Unique identifier for the device in Azure AD. | keyword | @@ -928,124 +843,14 @@ This is the `event` dataset. | m365_defender.event.user_level_policy | End-user mailbox policy that triggered the action taken on the email. | keyword | | m365_defender.event.vendor | Name of the product vendor or manufacturer, only available if device discovery finds enough information about this attribute. | keyword | | m365_defender.event.workload | The application from which the user clicked on the link, with the values being Email, Office and Teams. | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.version | Observer version. | keyword | | process.Ext.api.name | | keyword | | process.Ext.api.parameters.address | The target memory address. | long | | process.Ext.api.parameters.desired_access_numeric | This parameter indicates the numeric value of the `DesiredAccess` field passed to `OpenProcess` or `OpenThread`. | long | | process.Ext.api.parameters.protection | The memory protection for the region of pages. Corresponds to `MEMORY_BASIC_INFORMATION.Protect`. | keyword | | process.Ext.api.parameters.size | The size of parameter values passed to the API call. | long | | process.Ext.token.integrity_level_name | Integrity level that determine the levels of protection or access for a principal used by Mandatory Integrity Control (MIC). | keyword | -| process.args | Array of process arguments, starting with the absolute path to the executable. May be filtered to protect sensitive information. | keyword | -| process.args_count | Length of the process.args array. This field can be useful for querying or performing bucket analysis on how many arguments were provided to start a process. More arguments may be an indication of suspicious activity. | long | -| process.code_signature.status | Additional information about the certificate status. This is useful for logging cryptographic errors with the certificate validity or trust status. Leave unpopulated if the validity or trust of the certificate was unchecked. | keyword | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.executable | Absolute path to the process executable. | keyword | -| process.executable.text | Multi-field of `process.executable`. | match_only_text | -| process.hash.md5 | MD5 hash. | keyword | -| process.hash.sha1 | SHA1 hash. | keyword | -| process.hash.sha256 | SHA256 hash. | keyword | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.parent.args | Array of process arguments, starting with the absolute path to the executable. May be filtered to protect sensitive information. | keyword | -| process.parent.args_count | Length of the process.args array. This field can be useful for querying or performing bucket analysis on how many arguments were provided to start a process. More arguments may be an indication of suspicious activity. | long | -| process.parent.code_signature.exists | Boolean to capture if a signature is present. | boolean | -| process.parent.code_signature.status | Additional information about the certificate status. This is useful for logging cryptographic errors with the certificate validity or trust status. Leave unpopulated if the validity or trust of the certificate was unchecked. | keyword | -| process.parent.code_signature.trusted | Stores the trust status of the certificate chain. Validating the trust of the certificate chain may be complicated, and this field should only be populated by tools that actively check the status. | boolean | -| process.parent.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.parent.command_line.text | Multi-field of `process.parent.command_line`. | match_only_text | -| process.parent.executable | Absolute path to the process executable. | keyword | -| process.parent.executable.text | Multi-field of `process.parent.executable`. | match_only_text | | process.parent.group_leader.name | | keyword | -| process.parent.group_leader.pid | Process id. | long | -| process.parent.group_leader.start | The time the process started. | date | -| process.parent.hash.md5 | MD5 hash. | keyword | -| process.parent.hash.sha1 | SHA1 hash. | keyword | -| process.parent.hash.sha256 | SHA256 hash. | keyword | -| process.parent.name | Process name. Sometimes called program name or similar. | keyword | -| process.parent.name.text | Multi-field of `process.parent.name`. | match_only_text | -| process.parent.pe.company | Internal company name of the file, provided at compile-time. | keyword | -| process.parent.pe.description | Internal description of the file, provided at compile-time. | keyword | -| process.parent.pe.file_version | Internal version of the file, provided at compile-time. | keyword | -| process.parent.pe.original_file_name | Internal name of the file, provided at compile-time. | keyword | -| process.parent.pe.product | Internal product name of the file, provided at compile-time. | keyword | -| process.parent.pid | Process id. | long | -| process.parent.start | The time the process started. | date | -| process.pe.company | Internal company name of the file, provided at compile-time. | keyword | -| process.pe.description | Internal description of the file, provided at compile-time. | keyword | -| process.pe.file_version | Internal version of the file, provided at compile-time. | keyword | -| process.pe.original_file_name | Internal name of the file, provided at compile-time. | keyword | -| process.pe.product | Internal product name of the file, provided at compile-time. | keyword | -| process.pid | Process id. | long | -| process.start | The time the process started. | date | -| registry.data.strings | Content when writing string types. Populated as an array when writing string data to the registry. For single string registry types (REG_SZ, REG_EXPAND_SZ), this should be an array with one string. For sequences of string with REG_MULTI_SZ, this array will be variable length. For numeric data, such as REG_DWORD and REG_QWORD, this should be populated with the decimal representation (e.g `"1"`). | wildcard | -| registry.data.type | Standard registry type for encoding contents | keyword | -| registry.hive | Abbreviated name for the hive. | keyword | -| registry.key | Hive-relative path of keys. | keyword | -| registry.path | Full path, including hive, key and value | keyword | -| registry.value | Name of the value written. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| source.user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| source.user.id | Unique identifier of the user. | keyword | -| source.user.name | Short name or login of the user. | keyword | -| source.user.name.text | Multi-field of `source.user.name`. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| threat.group.name | The name of the group for a set of related intrusion activity that are tracked by a common name in the security community. While not required, you can use a MITRE ATT&CK® group name. | keyword | -| threat.indicator.file.directory | Directory where the file is located. It should include the drive letter, when appropriate. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.file.name | Name of the file including the extension, without the directory. | keyword | -| threat.indicator.file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| threat.indicator.registry.data.strings | Content when writing string types. Populated as an array when writing string data to the registry. For single string registry types (REG_SZ, REG_EXPAND_SZ), this should be an array with one string. For sequences of string with REG_MULTI_SZ, this array will be variable length. For numeric data, such as REG_DWORD and REG_QWORD, this should be populated with the decimal representation (e.g `"1"`). | wildcard | -| threat.indicator.registry.key | Hive-relative path of keys. | keyword | -| threat.indicator.registry.value | Name of the value written. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.technique.subtechnique.id | The full id of subtechnique used by this threat. You can use a MITRE ATT&CK® subtechnique, for example. (ex. https://attack.mitre.org/techniques/T1059/001/) | keyword | -| threat.technique.subtechnique.name | The name of subtechnique used by this threat. You can use a MITRE ATT&CK® subtechnique, for example. (ex. https://attack.mitre.org/techniques/T1059/001/) | keyword | -| threat.technique.subtechnique.name.text | Multi-field of `threat.technique.subtechnique.name`. | match_only_text | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.password | Password of the request. | keyword | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | | url.user_info | | keyword | -| url.username | Username of the request. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | ### incident @@ -1394,66 +1199,15 @@ An example event for `incident` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.delivery_timestamp | The date and time when the email message was received by the service or client. | date | -| email.direction | The direction of the message based on the sending and receiving domains. | keyword | -| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | -| email.subject | A brief summary of the topic of the message. | keyword | -| email.subject.text | Multi-field of `email.subject`. | match_only_text | -| email.to.address | The email address of recipient | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| event.url | URL linking to an external system to continue investigation of this event. This URL links to another system where in-depth investigation of the specific occurrence of this event can take place. Alert events, indicated by `event.kind:alert`, are a common use case for this field. | keyword | -| file.hash.sha1 | SHA1 hash. | keyword | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| group.id | Unique identifier for the group on the system/platform. | keyword | -| group.name | Name of the group. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | m365_defender.incident.alert.actor_display_name | The adversary or activity group that is associated with this alert. | keyword | @@ -1643,39 +1397,6 @@ An example event for `incident` looks as following: | m365_defender.incident.web_url.query | | keyword | | m365_defender.incident.web_url.scheme | | keyword | | m365_defender.incident.web_url.username | | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.hash.sha1 | SHA1 hash. | keyword | -| process.hash.sha256 | SHA256 hash. | keyword | -| process.parent.hash.sha1 | SHA1 hash. | keyword | -| process.parent.hash.sha256 | SHA256 hash. | keyword | -| process.parent.pid | Process id. | long | -| process.parent.start | The time the process started. | date | -| process.pid | Process id. | long | -| process.start | The time the process started. | date | -| process.user.id | Unique identifier of the user. | keyword | -| process.user.name | Short name or login of the user. | keyword | -| process.user.name.text | Multi-field of `process.user.name`. | match_only_text | -| registry.data.type | Standard registry type for encoding contents | keyword | -| registry.hive | Abbreviated name for the hive. | keyword | -| registry.key | Hive-relative path of keys. | keyword | -| registry.value | Name of the value written. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.user.name | Short name or login of the user. | keyword | -| source.user.name.text | Multi-field of `source.user.name`. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| threat.group.name | The name of the group for a set of related intrusion activity that are tracked by a common name in the security community. While not required, you can use a MITRE ATT&CK® group name. | keyword | -| threat.tactic.name | Name of the type of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/) | keyword | -| threat.technique.subtechnique.id | The full id of subtechnique used by this threat. You can use a MITRE ATT&CK® subtechnique, for example. (ex. https://attack.mitre.org/techniques/T1059/001/) | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | ### log @@ -1803,59 +1524,15 @@ An example event for `log` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.hash.sha1 | SHA1 hash. | keyword | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | m365_defender.alerts.actorName | The activity group, if any, the associated with this alert. | keyword | @@ -1908,37 +1585,4 @@ An example event for `log` looks as following: | m365_defender.redirectIncidentId | Only populated in case an incident is being grouped together with another incident, as part of the incident processing logic. | keyword | | m365_defender.status | Specifies the current status of the alert. Possible values are: 'Unknown', 'New', 'InProgress' and 'Resolved'. | keyword | | m365_defender.tags | Array of custom tags associated with an incident, for example to flag a group of incidents with a common characteristic. | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| observer.name | Custom name of the observer. This is a name that can be given to an observer. This can be helpful for example if multiple firewalls of the same model are used in an organization. If no custom name is needed, the field can be left empty. | keyword | -| observer.product | The product name of the observer. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.parent.pid | Process id. | long | -| process.parent.start | The time the process started. | date | -| process.pid | Process id. | long | -| process.start | The time the process started. | date | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.description | The description of the rule generating the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| threat.framework | Name of the threat framework used to further categorize and classify the tactic and technique of the reported threat. Framework classification can be provided by detecting systems, evaluated at ingest time, or retrospectively tagged to events. | keyword | -| threat.technique.name | The name of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.name.text | Multi-field of `threat.technique.name`. | match_only_text | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| url.full.text | Multi-field of `url.full`. | match_only_text | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | diff --git a/packages/m365_defender/manifest.yml b/packages/m365_defender/manifest.yml index d96266b8594..34ac3273997 100644 --- a/packages/m365_defender/manifest.yml +++ b/packages/m365_defender/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: m365_defender title: Microsoft M365 Defender -version: "2.12.0" +version: "2.13.0" description: Collect logs from Microsoft M365 Defender with Elastic Agent. categories: - "security" @@ -11,7 +11,7 @@ conditions: elastic: subscription: basic kibana: - version: ^8.12.0 + version: "^8.13.0" policy_templates: - name: m365_defender title: M365 Defender Logs From 402bbef5c94d5641eb87d279cc5f27fb80ae1f21 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:25:53 +0930 Subject: [PATCH 062/121] [mattermost] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^7.16.0 || ^8.0.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/mattermost --- packages/mattermost/changelog.yml | 5 + .../data_stream/audit/fields/agent.yml | 167 +----------------- .../data_stream/audit/fields/beats.yml | 3 - .../data_stream/audit/fields/ecs.yml | 74 -------- packages/mattermost/docs/README.md | 71 -------- packages/mattermost/manifest.yml | 4 +- 6 files changed, 8 insertions(+), 316 deletions(-) delete mode 100644 packages/mattermost/data_stream/audit/fields/ecs.yml diff --git a/packages/mattermost/changelog.yml b/packages/mattermost/changelog.yml index 9907ceedc28..8c39390a0fb 100644 --- a/packages/mattermost/changelog.yml +++ b/packages/mattermost/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.1.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.0.1" changes: - description: Fix sample event. diff --git a/packages/mattermost/data_stream/audit/fields/agent.yml b/packages/mattermost/data_stream/audit/fields/agent.yml index da4e652c53b..2bc58530bac 100644 --- a/packages/mattermost/data_stream/audit/fields/agent.yml +++ b/packages/mattermost/data_stream/audit/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/mattermost/data_stream/audit/fields/beats.yml b/packages/mattermost/data_stream/audit/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/mattermost/data_stream/audit/fields/beats.yml +++ b/packages/mattermost/data_stream/audit/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/mattermost/data_stream/audit/fields/ecs.yml b/packages/mattermost/data_stream/audit/fields/ecs.yml deleted file mode 100644 index 4ef73c50eee..00000000000 --- a/packages/mattermost/data_stream/audit/fields/ecs.yml +++ /dev/null @@ -1,74 +0,0 @@ -- name: user_agent.device.name - external: ecs -- name: user_agent.name - external: ecs -- name: user_agent.original - external: ecs -- name: user_agent.os.name - external: ecs -- name: user_agent.os.version - external: ecs -- name: user_agent.os.full - external: ecs -- name: user_agent.version - external: ecs -- name: url.path - external: ecs -- name: url.original - external: ecs -- name: source.address - external: ecs -- name: source.as.number - external: ecs -- name: source.as.organization.name - external: ecs -- name: source.bytes - external: ecs -- name: source.geo.city_name - external: ecs -- name: source.geo.continent_name - external: ecs -- name: source.geo.country_iso_code - external: ecs -- name: source.geo.country_name - external: ecs -- name: source.geo.location - external: ecs -- name: source.geo.name - external: ecs -- name: source.geo.region_iso_code - external: ecs -- name: source.geo.region_name - external: ecs -- name: source.ip - external: ecs -- name: tags - external: ecs -- name: ecs.version - external: ecs -- name: error.code - external: ecs -- name: group.id - external: ecs -- name: group.name - external: ecs -- name: http.response.status_code - external: ecs -- name: user.id - external: ecs -- name: user.target.id - external: ecs -- name: user.target.name - external: ecs -- name: user.target.roles - external: ecs -- name: user.target.group.id - external: ecs -- name: user.target.group.name - external: ecs -- name: user.changes.name - external: ecs -- name: related.user - external: ecs -- name: related.ip - external: ecs diff --git a/packages/mattermost/docs/README.md b/packages/mattermost/docs/README.md index 3f6ad1ba484..c0c2d19fc21 100644 --- a/packages/mattermost/docs/README.md +++ b/packages/mattermost/docs/README.md @@ -15,48 +15,16 @@ All access to the Mattermost REST API or CLI is audited. | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.code | Error code describing the error. | keyword | | event.dataset | Event dataset | constant_keyword | | event.module | Event module | constant_keyword | -| group.id | Unique identifier for the group on the system/platform. | keyword | -| group.name | Name of the group. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.response.status_code | HTTP response status code. | long | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | | mattermost.audit.api_path | REST API endpoint | keyword | @@ -79,45 +47,6 @@ All access to the Mattermost REST API or CLI is audited. | mattermost.audit.team.id | ID of affected team | keyword | | mattermost.audit.team.name | Name of affected team | keyword | | mattermost.audit.team.type | Type of affected team | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| user.changes.name | Short name or login of the user. | keyword | -| user.changes.name.text | Multi-field of `user.changes.name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | -| user.target.group.id | Unique identifier for the group on the system/platform. | keyword | -| user.target.group.name | Name of the group. | keyword | -| user.target.id | Unique identifier of the user. | keyword | -| user.target.name | Short name or login of the user. | keyword | -| user.target.name.text | Multi-field of `user.target.name`. | match_only_text | -| user.target.roles | Array of user roles at the time of the event. | keyword | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | An example event for `audit` looks as following: diff --git a/packages/mattermost/manifest.yml b/packages/mattermost/manifest.yml index dca2dafcc41..d556d7c21c2 100644 --- a/packages/mattermost/manifest.yml +++ b/packages/mattermost/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: mattermost title: "Mattermost" -version: "2.0.1" +version: "2.1.0" description: Collect logs from Mattermost with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - productivity_security conditions: kibana: - version: "^7.16.0 || ^8.0.0" + version: "^8.13.0" icons: - src: /img/mattermost-logo.svg title: Mattermost logo From 4bf6e532621621d09f4177130ba239bc548cdd57 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:25:55 +0930 Subject: [PATCH 063/121] [menlo] - Updated fields definitions Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/menlo --- packages/menlo/changelog.yml | 5 + .../menlo/data_stream/dlp/fields/agent.yml | 138 ------------------ packages/menlo/data_stream/dlp/fields/ecs.yml | 54 ------- .../menlo/data_stream/web/fields/agent.yml | 138 ------------------ packages/menlo/data_stream/web/fields/ecs.yml | 104 ------------- packages/menlo/docs/README.md | 134 ----------------- packages/menlo/manifest.yml | 2 +- 7 files changed, 6 insertions(+), 569 deletions(-) delete mode 100644 packages/menlo/data_stream/dlp/fields/ecs.yml delete mode 100644 packages/menlo/data_stream/web/fields/ecs.yml diff --git a/packages/menlo/changelog.yml b/packages/menlo/changelog.yml index 62c94d03acb..423c79404a2 100644 --- a/packages/menlo/changelog.yml +++ b/packages/menlo/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.1.0" + changes: + - description: Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.0.0" changes: - description: Release package as GA. diff --git a/packages/menlo/data_stream/dlp/fields/agent.yml b/packages/menlo/data_stream/dlp/fields/agent.yml index 98d2f9f38d5..894e6f12be2 100644 --- a/packages/menlo/data_stream/dlp/fields/agent.yml +++ b/packages/menlo/data_stream/dlp/fields/agent.yml @@ -5,153 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/menlo/data_stream/dlp/fields/ecs.yml b/packages/menlo/data_stream/dlp/fields/ecs.yml deleted file mode 100644 index bc427b45597..00000000000 --- a/packages/menlo/data_stream/dlp/fields/ecs.yml +++ /dev/null @@ -1,54 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.action -- external: ecs - name: event.id -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.severity -- external: ecs - name: file.hash.sha256 -- external: ecs - name: file.name -- external: ecs - name: http.request.method -- external: ecs - name: observer.product -- external: ecs - name: observer.vendor -- external: ecs - name: observer.version -- external: ecs - name: related.user -- external: ecs - name: related.hash -- external: ecs - name: rule.id -- external: ecs - name: rule.name -- external: ecs - name: tags -- external: ecs - name: url.domain -- external: ecs - name: url.original -- external: ecs - name: url.path -- external: ecs - name: url.registered_domain -- external: ecs - name: url.scheme -- external: ecs - name: url.top_level_domain -- external: ecs - name: user.name diff --git a/packages/menlo/data_stream/web/fields/agent.yml b/packages/menlo/data_stream/web/fields/agent.yml index 98d2f9f38d5..894e6f12be2 100644 --- a/packages/menlo/data_stream/web/fields/agent.yml +++ b/packages/menlo/data_stream/web/fields/agent.yml @@ -5,153 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/menlo/data_stream/web/fields/ecs.yml b/packages/menlo/data_stream/web/fields/ecs.yml deleted file mode 100644 index 675463fef80..00000000000 --- a/packages/menlo/data_stream/web/fields/ecs.yml +++ /dev/null @@ -1,104 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.type -- external: ecs - name: destination.domain -- external: ecs - name: error.message -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.reason -- external: ecs - name: event.severity -- external: ecs - name: dns.answers.data -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.ip -- external: ecs - name: client.geo.country_iso_code -- external: ecs - name: client.ip -- external: ecs - name: network.protocol -- external: ecs - name: observer.geo.country_iso_code -- external: ecs - name: observer.ip -- external: ecs - name: observer.product -- external: ecs - name: observer.vendor -- external: ecs - name: observer.version -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: related.hash -- external: ecs - name: file.name -- external: ecs - name: http.request.method -- external: ecs - name: http.request.mime_type -- external: ecs - name: http.request.referrer -- external: ecs - name: http.response.status_code -- external: ecs - name: message -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.ip -- external: ecs - name: server.geo.country_iso_code -- external: ecs - name: server.ip -- external: ecs - name: tags -- external: ecs - name: url.domain -- external: ecs - name: url.registered_domain -- external: ecs - name: url.top_level_domain -- external: ecs - name: url.original -- external: ecs - name: url.path -- external: ecs - name: url.scheme -- external: ecs - name: url.subdomain -- external: ecs - name: user.name -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version diff --git a/packages/menlo/docs/README.md b/packages/menlo/docs/README.md index e9189d2250d..870e68000e5 100644 --- a/packages/menlo/docs/README.md +++ b/packages/menlo/docs/README.md @@ -192,61 +192,15 @@ An example event for `web` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| client.geo.country_iso_code | Country ISO code. | keyword | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| dns.answers.data | The data describing the resource. The meaning of this data depends on the type and class of the resource record. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.reason | Reason why this event happened, according to the source. This describes the why of a particular action or outcome captured in the event. Where `event.action` captures the action from the event, `event.reason` describes why that action was taken. For example, a web proxy with an `event.action` which denied the request may also populate `event.reason` with the reason why (e.g. `blocked site`). | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.request.mime_type | Mime type of the body of the request. This value must only be populated based on the content of the request body, not on the `Content-Type` header. Comparing the mime type of a request with the request's Content-Type header can be helpful in detecting threats or misconfigured clients. | keyword | -| http.request.referrer | Referrer for this HTTP request. | keyword | -| http.response.status_code | HTTP response status code. | long | | input.type | Input type | keyword | | log.offset | Log offset | long | | menlo.web.cached | Indicates whether the resource was obtained from the isolated browser’s cache (True) or by downloading from the origin server (False) | boolean | @@ -274,41 +228,6 @@ An example event for `web` looks as following: | menlo.web.ua_type | The type of user agent | keyword | | menlo.web.virus_details | Virus detail | keyword | | menlo.web.xff_ip | X-Forwarded-For HTTP header field originating client IP address | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| observer.geo.country_iso_code | Country ISO code. | keyword | -| observer.ip | IP addresses of the observer. | ip | -| observer.product | The product name of the observer. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| observer.version | Observer version. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| server.geo.country_iso_code | Country ISO code. | keyword | -| server.ip | IP address of the server (IPv4 or IPv6). | ip | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | ### DLP @@ -425,51 +344,15 @@ An example event for `dlp` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | menlo.dlp.alerted | Whether or not an email alert was sent to a DLP Auditor profile | boolean | @@ -480,21 +363,4 @@ An example event for `dlp` looks as following: | menlo.dlp.status | Result from the DLP engine | keyword | | menlo.dlp.stream_name | Internal name used for the file (usually working_file) or text stream (uid) | keyword | | menlo.dlp.user_input | Whether or not this event was generated as a result of user form input | boolean | -| observer.product | The product name of the observer. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| observer.version | Observer version. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | diff --git a/packages/menlo/manifest.yml b/packages/menlo/manifest.yml index 3bfc6efca3d..e93da73be5a 100644 --- a/packages/menlo/manifest.yml +++ b/packages/menlo/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: menlo title: "Menlo Security" -version: 1.0.0 +version: "1.1.0" source: license: "Elastic-2.0" description: "Collect logs from Menlo Security products with Elastic Agent" From 9a3922d94f3229bcacdcbf34550fa15e25f9b69f Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:25:56 +0930 Subject: [PATCH 064/121] [microsoft_defender_cloud] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/microsoft_defender_cloud --- packages/microsoft_defender_cloud/_dev/build/build.yml | 1 - packages/microsoft_defender_cloud/changelog.yml | 5 +++++ .../data_stream/event/fields/beats.yml | 3 --- packages/microsoft_defender_cloud/docs/README.md | 1 - packages/microsoft_defender_cloud/manifest.yml | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/microsoft_defender_cloud/_dev/build/build.yml b/packages/microsoft_defender_cloud/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/microsoft_defender_cloud/_dev/build/build.yml +++ b/packages/microsoft_defender_cloud/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/microsoft_defender_cloud/changelog.yml b/packages/microsoft_defender_cloud/changelog.yml index e38d7b2042b..0c541b0cec0 100644 --- a/packages/microsoft_defender_cloud/changelog.yml +++ b/packages/microsoft_defender_cloud/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.2.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.1.2" changes: - description: Fix name canonicalization routines. diff --git a/packages/microsoft_defender_cloud/data_stream/event/fields/beats.yml b/packages/microsoft_defender_cloud/data_stream/event/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/microsoft_defender_cloud/data_stream/event/fields/beats.yml +++ b/packages/microsoft_defender_cloud/data_stream/event/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/microsoft_defender_cloud/docs/README.md b/packages/microsoft_defender_cloud/docs/README.md index 28c460ada13..af4046f448a 100644 --- a/packages/microsoft_defender_cloud/docs/README.md +++ b/packages/microsoft_defender_cloud/docs/README.md @@ -294,5 +294,4 @@ This is the `Event` dataset. | microsoft_defender_cloud.event.workspace.id | | keyword | | microsoft_defender_cloud.event.workspace.resource_group | | keyword | | microsoft_defender_cloud.event.workspace.subscription_id | | keyword | -| tags | User defined tags. | keyword | diff --git a/packages/microsoft_defender_cloud/manifest.yml b/packages/microsoft_defender_cloud/manifest.yml index 0e3033531da..4eee3715cce 100644 --- a/packages/microsoft_defender_cloud/manifest.yml +++ b/packages/microsoft_defender_cloud/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: microsoft_defender_cloud title: Microsoft Defender for Cloud -version: "1.1.2" +version: "1.2.0" description: Collect logs from Microsoft Defender for Cloud with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - cloudsecurity_cdr conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: subscription: basic screenshots: From cd5beb53e0eb092b151d41e3c44e2c66a034a93f Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:25:57 +0930 Subject: [PATCH 065/121] [microsoft_defender_endpoint] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/microsoft_defender_endpoint --- .../microsoft_defender_endpoint/changelog.yml | 5 + .../data_stream/log/fields/agent.yml | 167 +----------------- .../data_stream/log/fields/ecs.yml | 92 ---------- .../docs/README.md | 76 -------- .../microsoft_defender_endpoint/manifest.yml | 4 +- 5 files changed, 8 insertions(+), 336 deletions(-) delete mode 100644 packages/microsoft_defender_endpoint/data_stream/log/fields/ecs.yml diff --git a/packages/microsoft_defender_endpoint/changelog.yml b/packages/microsoft_defender_endpoint/changelog.yml index f8314cd2924..f8335597db2 100644 --- a/packages/microsoft_defender_endpoint/changelog.yml +++ b/packages/microsoft_defender_endpoint/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.25.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.24.2" changes: - description: Fix bug handling message field when events are received from Logstash with `ecs_compatibility` turned on. diff --git a/packages/microsoft_defender_endpoint/data_stream/log/fields/agent.yml b/packages/microsoft_defender_endpoint/data_stream/log/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/microsoft_defender_endpoint/data_stream/log/fields/agent.yml +++ b/packages/microsoft_defender_endpoint/data_stream/log/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/microsoft_defender_endpoint/data_stream/log/fields/ecs.yml b/packages/microsoft_defender_endpoint/data_stream/log/fields/ecs.yml deleted file mode 100644 index 6f8b6071989..00000000000 --- a/packages/microsoft_defender_endpoint/data_stream/log/fields/ecs.yml +++ /dev/null @@ -1,92 +0,0 @@ -- external: ecs - name: container.image.tag -- external: ecs - name: container.runtime -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.duration -- external: ecs - name: event.end -- external: ecs - name: event.id -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.provider -- external: ecs - name: event.severity -- external: ecs - name: event.start -- external: ecs - name: event.type -- external: ecs - name: file.extension -- external: ecs - name: file.hash.md5 -- external: ecs - name: file.hash.sha1 -- external: ecs - name: file.hash.sha256 -- external: ecs - name: file.hash.sha512 -- external: ecs - name: file.name -- external: ecs - name: file.path -- external: ecs - name: log.file.path -- external: ecs - name: log.logger -- external: ecs - name: message -- external: ecs - name: observer.name -- external: ecs - name: observer.product -- external: ecs - name: observer.type -- external: ecs - name: observer.vendor -- external: ecs - name: process.command_line -- external: ecs - name: process.parent.pid -- external: ecs - name: process.parent.start -- external: ecs - name: process.pid -- external: ecs - name: process.start -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: rule.description -- external: ecs - name: tags -- external: ecs - name: threat.framework -- external: ecs - name: threat.technique.name -- external: ecs - name: user.domain -- external: ecs - name: user.id -- external: ecs - name: user.name diff --git a/packages/microsoft_defender_endpoint/docs/README.md b/packages/microsoft_defender_endpoint/docs/README.md index 2ea1439dd51..6143851476b 100644 --- a/packages/microsoft_defender_endpoint/docs/README.md +++ b/packages/microsoft_defender_endpoint/docs/README.md @@ -164,70 +164,17 @@ An example event for `log` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.image.tag | Container image tags. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| container.runtime | Runtime managing this container. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.duration | Duration of the event in nanoseconds. If `event.start` and `event.end` are known this value should be the difference between the end and start time. | long | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| file.hash.md5 | MD5 hash. | keyword | -| file.hash.sha1 | SHA1 hash. | keyword | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.hash.sha512 | SHA512 hash. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | -| log.logger | The name of the logger inside an application. This is usually the name of the class which initialized the logger, or can be a custom name. | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | microsoft.defender_endpoint.assignedTo | Owner of the alert. | keyword | | microsoft.defender_endpoint.classification | Specification of the alert. Possible values are: 'Unknown', 'FalsePositive', 'TruePositive'. | keyword | | microsoft.defender_endpoint.determination | Specifies the determination of the alert. Possible values are: 'NotAvailable', 'Apt', 'Malware', 'SecurityPersonnel', 'SecurityTesting', 'UnwantedSoftware', 'Other'. | keyword | @@ -245,27 +192,4 @@ An example event for `log` looks as following: | microsoft.defender_endpoint.resolvedTime | The date and time in which the status of the alert was changed to 'Resolved'. | date | | microsoft.defender_endpoint.status | Specifies the current status of the alert. Possible values are: 'Unknown', 'New', 'InProgress' and 'Resolved'. | keyword | | microsoft.defender_endpoint.threatFamilyName | Threat family. | keyword | -| observer.name | Custom name of the observer. This is a name that can be given to an observer. This can be helpful for example if multiple firewalls of the same model are used in an organization. If no custom name is needed, the field can be left empty. | keyword | -| observer.product | The product name of the observer. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.parent.pid | Process id. | long | -| process.parent.start | The time the process started. | date | -| process.pid | Process id. | long | -| process.start | The time the process started. | date | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.description | The description of the rule generating the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| threat.framework | Name of the threat framework used to further categorize and classify the tactic and technique of the reported threat. Framework classification can be provided by detecting systems, evaluated at ingest time, or retrospectively tagged to events. | keyword | -| threat.technique.name | The name of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.name.text | Multi-field of `threat.technique.name`. | match_only_text | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | diff --git a/packages/microsoft_defender_endpoint/manifest.yml b/packages/microsoft_defender_endpoint/manifest.yml index bc33e45e7d0..121db9fcedb 100644 --- a/packages/microsoft_defender_endpoint/manifest.yml +++ b/packages/microsoft_defender_endpoint/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: microsoft_defender_endpoint title: Microsoft Defender for Endpoint -version: "2.24.2" +version: "2.25.0" description: Collect logs from Microsoft Defender for Endpoint with Elastic Agent. categories: - "security" @@ -9,7 +9,7 @@ categories: type: integration conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" policy_templates: - name: microsoft_defender_endpoint title: Microsoft Defender for Endpoint From 5ca947411f62e07bf4af6eefbd2112c4b5dbf8e6 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:25:58 +0930 Subject: [PATCH 066/121] [microsoft_exchange_online_message_trace] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/microsoft_exchange_online_message_trace --- .../changelog.yml | 5 + .../data_stream/log/fields/ecs.yml | 114 ------------------ .../docs/README.md | 61 ---------- .../manifest.yml | 4 +- 4 files changed, 7 insertions(+), 177 deletions(-) delete mode 100644 packages/microsoft_exchange_online_message_trace/data_stream/log/fields/ecs.yml diff --git a/packages/microsoft_exchange_online_message_trace/changelog.yml b/packages/microsoft_exchange_online_message_trace/changelog.yml index 0732d7b3822..35ffe803471 100644 --- a/packages/microsoft_exchange_online_message_trace/changelog.yml +++ b/packages/microsoft_exchange_online_message_trace/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.22.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.21.2" changes: - description: Fix template to not fail without local domains. diff --git a/packages/microsoft_exchange_online_message_trace/data_stream/log/fields/ecs.yml b/packages/microsoft_exchange_online_message_trace/data_stream/log/fields/ecs.yml deleted file mode 100644 index f7117c7dfb2..00000000000 --- a/packages/microsoft_exchange_online_message_trace/data_stream/log/fields/ecs.yml +++ /dev/null @@ -1,114 +0,0 @@ -- external: ecs - name: email.attachments.file.size -- external: ecs - name: email.delivery_timestamp -- external: ecs - name: email.from.address -- external: ecs - name: email.local_id -- external: ecs - name: email.message_id -- external: ecs - name: email.subject -- external: ecs - name: email.to.address -- external: ecs - name: ecs.version -- external: ecs - name: tags -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.as.number -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- description: Longitude and latitude. - level: core - name: destination.geo.location - type: geo_point -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.ip -- external: ecs - name: email.direction -- external: ecs - name: log.file.path -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.as.number -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- description: Longitude and latitude. - level: core - name: source.geo.location - type: geo_point -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: event.created -- external: ecs - name: event.start -- external: ecs - name: event.end -- external: ecs - name: destination.domain -- external: ecs - name: destination.registered_domain -- external: ecs - name: destination.top_level_domain -- external: ecs - name: destination.subdomain -- external: ecs - name: source.domain -- external: ecs - name: source.registered_domain -- external: ecs - name: source.top_level_domain -- external: ecs - name: source.subdomain -- external: ecs - name: user.name -- external: ecs - name: user.domain -- external: ecs - name: source.user.domain -- external: ecs - name: source.user.email -- external: ecs - name: source.user.id -- external: ecs - name: source.user.name -- external: ecs - name: user.email -- external: ecs - name: user.id -- external: ecs - name: destination.user.email -- external: ecs - name: destination.user.name -- external: ecs - name: destination.user.domain -- external: ecs - name: destination.user.id -- external: ecs - name: related.user diff --git a/packages/microsoft_exchange_online_message_trace/docs/README.md b/packages/microsoft_exchange_online_message_trace/docs/README.md index 4e2a675dde2..54da485d4e5 100644 --- a/packages/microsoft_exchange_online_message_trace/docs/README.md +++ b/packages/microsoft_exchange_online_message_trace/docs/README.md @@ -269,42 +269,8 @@ An example event for `log` looks as following: | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.registered_domain | The highest registered destination domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| destination.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| destination.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| destination.user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| destination.user.email | User email address. | keyword | -| destination.user.id | Unique identifier of the user. | keyword | -| destination.user.name | Short name or login of the user. | keyword | -| destination.user.name.text | Multi-field of `destination.user.name`. | match_only_text | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.attachments.file.size | Attachment file size in bytes. | long | -| email.delivery_timestamp | The date and time when the email message was received by the service or client. | date | -| email.direction | The direction of the message based on the sending and receiving domains. | keyword | -| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | -| email.local_id | Unique identifier given to the email by the source that created the event. Identifier is not persistent across hops. | keyword | -| email.message_id | Identifier from the RFC 5322 `Message-ID:` email header that refers to a particular email message. | wildcard | -| email.subject | A brief summary of the topic of the message. | keyword | -| email.subject.text | Multi-field of `email.subject`. | match_only_text | -| email.to.address | The email address of recipient | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | | input.type | | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | | long | | microsoft.online_message_trace.EndDate | This field is used to limit the report period. Use this field in a $filter query option to set the end date and time of the reporting period. If you supply EndDate in the $filter option, you must also supply StartDate. In this report, this field corresponds to the date and time of the last processing step recorded for the message. | date_nanos | | microsoft.online_message_trace.FromIP | The IPv4 or IPv6 address that transmitted the message to the Office 365 email system. | keyword | @@ -320,30 +286,3 @@ An example event for `log` looks as following: | microsoft.online_message_trace.Status | The status of the message in the Office 365 email system. This corresponds to the Detail field of the last processing step recorded for the message.\\ | keyword | | microsoft.online_message_trace.Subject | The subject line of the message, if one was present for the message.\\ | keyword | | microsoft.online_message_trace.ToIP | The IPv4 or IPv6 address that the Office 365 email system sent the message to.\\ | keyword | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.registered_domain | The highest registered source domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| source.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| source.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| source.user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| source.user.email | User email address. | keyword | -| source.user.id | Unique identifier of the user. | keyword | -| source.user.name | Short name or login of the user. | keyword | -| source.user.name.text | Multi-field of `source.user.name`. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | diff --git a/packages/microsoft_exchange_online_message_trace/manifest.yml b/packages/microsoft_exchange_online_message_trace/manifest.yml index bda6c474bdc..27d503bd52b 100644 --- a/packages/microsoft_exchange_online_message_trace/manifest.yml +++ b/packages/microsoft_exchange_online_message_trace/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: microsoft_exchange_online_message_trace title: "Microsoft Exchange Online Message Trace" -version: "1.21.2" +version: "1.22.0" description: "Microsoft Exchange Online Message Trace Integration" type: integration categories: @@ -9,7 +9,7 @@ categories: - email_security conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" elastic: subscription: "basic" icons: From 9a99af15a69409cda315027bc2de6811ddcad59c Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:05 +0930 Subject: [PATCH 067/121] [mimecast] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/mimecast --- packages/mimecast/changelog.yml | 5 + .../archive_search_logs/fields/agent.yml | 151 ------- .../archive_search_logs/fields/ecs.yml | 22 - .../data_stream/audit_events/fields/agent.yml | 167 +------ .../data_stream/audit_events/fields/ecs.yml | 58 --- .../data_stream/dlp_logs/fields/agent.yml | 167 +------ .../data_stream/dlp_logs/fields/ecs.yml | 22 - .../data_stream/siem_logs/fields/agent.yml | 167 +------ .../data_stream/siem_logs/fields/ecs.yml | 82 ---- .../fields/agent.yml | 167 +------ .../fields/ecs.yml | 30 -- .../fields/agent.yml | 167 +------ .../threat_intel_malware_grid/fields/ecs.yml | 30 -- .../data_stream/ttp_ap_logs/fields/agent.yml | 167 +------ .../data_stream/ttp_ap_logs/fields/ecs.yml | 32 -- .../data_stream/ttp_ip_logs/fields/agent.yml | 167 +------ .../data_stream/ttp_ip_logs/fields/ecs.yml | 28 -- .../data_stream/ttp_url_logs/fields/agent.yml | 167 +------ .../data_stream/ttp_url_logs/fields/ecs.yml | 32 -- packages/mimecast/docs/README.md | 414 ------------------ packages/mimecast/manifest.yml | 4 +- 21 files changed, 15 insertions(+), 2231 deletions(-) delete mode 100644 packages/mimecast/data_stream/archive_search_logs/fields/ecs.yml delete mode 100644 packages/mimecast/data_stream/audit_events/fields/ecs.yml delete mode 100644 packages/mimecast/data_stream/dlp_logs/fields/ecs.yml delete mode 100644 packages/mimecast/data_stream/siem_logs/fields/ecs.yml delete mode 100644 packages/mimecast/data_stream/threat_intel_malware_customer/fields/ecs.yml delete mode 100644 packages/mimecast/data_stream/threat_intel_malware_grid/fields/ecs.yml delete mode 100644 packages/mimecast/data_stream/ttp_ap_logs/fields/ecs.yml delete mode 100644 packages/mimecast/data_stream/ttp_ip_logs/fields/ecs.yml delete mode 100644 packages/mimecast/data_stream/ttp_url_logs/fields/ecs.yml diff --git a/packages/mimecast/changelog.yml b/packages/mimecast/changelog.yml index fe27f64a68e..fa323cd6ace 100644 --- a/packages/mimecast/changelog.yml +++ b/packages/mimecast/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.26.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.25.0" changes: - description: Improve handling of empty responses. diff --git a/packages/mimecast/data_stream/archive_search_logs/fields/agent.yml b/packages/mimecast/data_stream/archive_search_logs/fields/agent.yml index c46a152ef14..894e6f12be2 100644 --- a/packages/mimecast/data_stream/archive_search_logs/fields/agent.yml +++ b/packages/mimecast/data_stream/archive_search_logs/fields/agent.yml @@ -5,166 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/mimecast/data_stream/archive_search_logs/fields/ecs.yml b/packages/mimecast/data_stream/archive_search_logs/fields/ecs.yml deleted file mode 100644 index 571dd97bf81..00000000000 --- a/packages/mimecast/data_stream/archive_search_logs/fields/ecs.yml +++ /dev/null @@ -1,22 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.action -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.original -- external: ecs - name: event.reason -- external: ecs - name: related.user -- external: ecs - name: tags -- external: ecs - name: user.domain -- external: ecs - name: user.email -- external: ecs - name: user.name diff --git a/packages/mimecast/data_stream/audit_events/fields/agent.yml b/packages/mimecast/data_stream/audit_events/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/mimecast/data_stream/audit_events/fields/agent.yml +++ b/packages/mimecast/data_stream/audit_events/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/mimecast/data_stream/audit_events/fields/ecs.yml b/packages/mimecast/data_stream/audit_events/fields/ecs.yml deleted file mode 100644 index 9105b62ec94..00000000000 --- a/packages/mimecast/data_stream/audit_events/fields/ecs.yml +++ /dev/null @@ -1,58 +0,0 @@ -- external: ecs - name: client.as.number -- external: ecs - name: client.as.organization.name -- external: ecs - name: client.geo.city_name -- external: ecs - name: client.geo.continent_name -- external: ecs - name: client.geo.country_iso_code -- external: ecs - name: client.geo.country_name -- external: ecs - name: client.geo.location -- external: ecs - name: client.geo.region_iso_code -- external: ecs - name: client.geo.region_name -- external: ecs - name: client.ip -- external: ecs - name: ecs.version -- external: ecs - name: email.from.address -- external: ecs - name: email.origination_timestamp -- external: ecs - name: email.subject -- external: ecs - name: email.to.address -- external: ecs - name: event.action -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.original -- external: ecs - name: event.reason -- external: ecs - name: file.extension -- external: ecs - name: file.name -- external: ecs - name: file.size -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: tags -- external: ecs - name: user.domain -- external: ecs - name: user.email -- external: ecs - name: user.name diff --git a/packages/mimecast/data_stream/dlp_logs/fields/agent.yml b/packages/mimecast/data_stream/dlp_logs/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/mimecast/data_stream/dlp_logs/fields/agent.yml +++ b/packages/mimecast/data_stream/dlp_logs/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/mimecast/data_stream/dlp_logs/fields/ecs.yml b/packages/mimecast/data_stream/dlp_logs/fields/ecs.yml deleted file mode 100644 index ef925714f24..00000000000 --- a/packages/mimecast/data_stream/dlp_logs/fields/ecs.yml +++ /dev/null @@ -1,22 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: email.direction -- external: ecs - name: email.from.address -- external: ecs - name: email.message_id -- external: ecs - name: email.subject -- external: ecs - name: email.to.address -- external: ecs - name: event.action -- external: ecs - name: event.created -- external: ecs - name: event.original -- external: ecs - name: rule.name -- external: ecs - name: tags diff --git a/packages/mimecast/data_stream/siem_logs/fields/agent.yml b/packages/mimecast/data_stream/siem_logs/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/mimecast/data_stream/siem_logs/fields/agent.yml +++ b/packages/mimecast/data_stream/siem_logs/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/mimecast/data_stream/siem_logs/fields/ecs.yml b/packages/mimecast/data_stream/siem_logs/fields/ecs.yml deleted file mode 100644 index 863be6474cd..00000000000 --- a/packages/mimecast/data_stream/siem_logs/fields/ecs.yml +++ /dev/null @@ -1,82 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: email.attachments.file.extension -- external: ecs - name: email.attachments.file.hash.md5 -- external: ecs - name: email.attachments.file.hash.sha1 -- external: ecs - name: email.attachments.file.hash.sha256 -- external: ecs - name: email.attachments.file.mime_type -- external: ecs - name: email.attachments.file.name -- external: ecs - name: email.attachments.file.size -- external: ecs - name: email.direction -- external: ecs - name: email.from.address -- external: ecs - name: email.local_id -- external: ecs - name: email.message_id -- external: ecs - name: email.subject -- external: ecs - name: email.to.address -- external: ecs - name: error.code -- external: ecs - name: error.message -- external: ecs - name: error.type -- external: ecs - name: event.action -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.reason -- external: ecs - name: rule.name -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.domain -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: tags -- external: ecs - name: tls.cipher -- external: ecs - name: tls.established -- external: ecs - name: tls.version -- external: ecs - name: url.full -- external: ecs - name: user.email diff --git a/packages/mimecast/data_stream/threat_intel_malware_customer/fields/agent.yml b/packages/mimecast/data_stream/threat_intel_malware_customer/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/mimecast/data_stream/threat_intel_malware_customer/fields/agent.yml +++ b/packages/mimecast/data_stream/threat_intel_malware_customer/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/mimecast/data_stream/threat_intel_malware_customer/fields/ecs.yml b/packages/mimecast/data_stream/threat_intel_malware_customer/fields/ecs.yml deleted file mode 100644 index 3c764373326..00000000000 --- a/packages/mimecast/data_stream/threat_intel_malware_customer/fields/ecs.yml +++ /dev/null @@ -1,30 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: message -- external: ecs - name: related.hash -- external: ecs - name: tags -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.first_seen -- external: ecs - name: threat.indicator.modified_at -- external: ecs - name: threat.indicator.type diff --git a/packages/mimecast/data_stream/threat_intel_malware_grid/fields/agent.yml b/packages/mimecast/data_stream/threat_intel_malware_grid/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/mimecast/data_stream/threat_intel_malware_grid/fields/agent.yml +++ b/packages/mimecast/data_stream/threat_intel_malware_grid/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/mimecast/data_stream/threat_intel_malware_grid/fields/ecs.yml b/packages/mimecast/data_stream/threat_intel_malware_grid/fields/ecs.yml deleted file mode 100644 index 3c764373326..00000000000 --- a/packages/mimecast/data_stream/threat_intel_malware_grid/fields/ecs.yml +++ /dev/null @@ -1,30 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: message -- external: ecs - name: related.hash -- external: ecs - name: tags -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.first_seen -- external: ecs - name: threat.indicator.modified_at -- external: ecs - name: threat.indicator.type diff --git a/packages/mimecast/data_stream/ttp_ap_logs/fields/agent.yml b/packages/mimecast/data_stream/ttp_ap_logs/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/mimecast/data_stream/ttp_ap_logs/fields/agent.yml +++ b/packages/mimecast/data_stream/ttp_ap_logs/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/mimecast/data_stream/ttp_ap_logs/fields/ecs.yml b/packages/mimecast/data_stream/ttp_ap_logs/fields/ecs.yml deleted file mode 100644 index d942cd864e0..00000000000 --- a/packages/mimecast/data_stream/ttp_ap_logs/fields/ecs.yml +++ /dev/null @@ -1,32 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: email.attachments.file.extension -- external: ecs - name: email.attachments.file.hash.sha256 -- external: ecs - name: email.attachments.file.mime_type -- external: ecs - name: email.attachments.file.name -- external: ecs - name: email.direction -- external: ecs - name: email.from.address -- external: ecs - name: email.message_id -- external: ecs - name: email.subject -- external: ecs - name: email.to.address -- external: ecs - name: event.action -- external: ecs - name: event.created -- external: ecs - name: event.original -- external: ecs - name: related.hash -- external: ecs - name: rule.name -- external: ecs - name: tags diff --git a/packages/mimecast/data_stream/ttp_ip_logs/fields/agent.yml b/packages/mimecast/data_stream/ttp_ip_logs/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/mimecast/data_stream/ttp_ip_logs/fields/agent.yml +++ b/packages/mimecast/data_stream/ttp_ip_logs/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/mimecast/data_stream/ttp_ip_logs/fields/ecs.yml b/packages/mimecast/data_stream/ttp_ip_logs/fields/ecs.yml deleted file mode 100644 index ae101f9d829..00000000000 --- a/packages/mimecast/data_stream/ttp_ip_logs/fields/ecs.yml +++ /dev/null @@ -1,28 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: email.from.address -- external: ecs - name: email.message_id -- external: ecs - name: email.subject -- external: ecs - name: email.to.address -- external: ecs - name: event.action -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.original -- external: ecs - name: related.ip -- external: ecs - name: rule.name -- external: ecs - name: source.domain -- external: ecs - name: source.ip -- external: ecs - name: tags diff --git a/packages/mimecast/data_stream/ttp_url_logs/fields/agent.yml b/packages/mimecast/data_stream/ttp_url_logs/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/mimecast/data_stream/ttp_url_logs/fields/agent.yml +++ b/packages/mimecast/data_stream/ttp_url_logs/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/mimecast/data_stream/ttp_url_logs/fields/ecs.yml b/packages/mimecast/data_stream/ttp_url_logs/fields/ecs.yml deleted file mode 100644 index faf406570c5..00000000000 --- a/packages/mimecast/data_stream/ttp_url_logs/fields/ecs.yml +++ /dev/null @@ -1,32 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: email.direction -- external: ecs - name: email.from.address -- external: ecs - name: email.message_id -- external: ecs - name: email.subject -- external: ecs - name: email.to.address -- external: ecs - name: event.action -- external: ecs - name: event.created -- external: ecs - name: event.original -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: rule.name -- external: ecs - name: source.ip -- external: ecs - name: tags -- external: ecs - name: url.original -- external: ecs - name: user.email diff --git a/packages/mimecast/docs/README.md b/packages/mimecast/docs/README.md index 00dcd52b0c5..e3d61881156 100644 --- a/packages/mimecast/docs/README.md +++ b/packages/mimecast/docs/README.md @@ -94,47 +94,15 @@ An example event for `archive_search` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.reason | Reason why this event happened, according to the source. This describes the why of a particular action or outcome captured in the event. Where `event.action` captures the action from the event, `event.reason` describes why that action was taken. For example, a web proxy with an `event.action` which denied the request may also populate `event.reason` with the reason why (e.g. `blocked site`). | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | mimecast.email.address | The email address of the user who performed the search. | keyword | @@ -143,12 +111,6 @@ An example event for `archive_search` looks as following: | mimecast.search_details.reason | The search reason entered when the search was executed if any. | keyword | | mimecast.search_details.source | The search source context | keyword | | mimecast.search_details.text | The text used in the search. | keyword | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.email | User email address. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | ### Audit Events @@ -223,66 +185,15 @@ An example event for `audit_events` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| client.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| client.as.organization.name | Organization name. | keyword | -| client.as.organization.name.text | Multi-field of `client.as.organization.name`. | match_only_text | -| client.geo.city_name | City name. | keyword | -| client.geo.continent_name | Name of the continent. | keyword | -| client.geo.country_iso_code | Country ISO code. | keyword | -| client.geo.country_name | Country name. | keyword | -| client.geo.location | Longitude and latitude. | geo_point | -| client.geo.region_iso_code | Region ISO code. | keyword | -| client.geo.region_name | Region name. | keyword | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | -| email.origination_timestamp | The date and time the email message was composed. Many email clients will fill in this value automatically when the message is sent by a user. | date | -| email.subject | A brief summary of the topic of the message. | keyword | -| email.subject.text | Multi-field of `email.subject`. | match_only_text | -| email.to.address | The email address of recipient | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.reason | Reason why this event happened, according to the source. This describes the why of a particular action or outcome captured in the event. Where `event.action` captures the action from the event, `event.reason` describes why that action was taken. For example, a web proxy with an `event.action` which denied the request may also populate `event.reason` with the reason why (e.g. `blocked site`). | keyword | -| file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | mimecast.2FA | Info about two-factor authentication. | keyword | @@ -294,13 +205,6 @@ An example event for `audit_events` looks as following: | mimecast.method | Method which triggers audit events. | keyword | | mimecast.remote | Info about remote IP trying to access the API. | keyword | | mimecast.remote_ip | Remote IP. | ip | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.email | User email address. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | ### DLP Logs @@ -377,55 +281,17 @@ An example event for `dlp` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.direction | The direction of the message based on the sending and receiving domains. | keyword | -| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | -| email.message_id | Identifier from the RFC 5322 `Message-ID:` email header that refers to a particular email message. | wildcard | -| email.subject | A brief summary of the topic of the message. | keyword | -| email.subject.text | Multi-field of `email.subject`. | match_only_text | -| email.to.address | The email address of recipient | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| rule.name | The name of the rule or signature generating the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | ### SIEM Logs @@ -504,65 +370,15 @@ An example event for `siem` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.attachments.file.extension | Attachment file extension, excluding the leading dot. | keyword | -| email.attachments.file.hash.md5 | MD5 hash. | keyword | -| email.attachments.file.hash.sha1 | SHA1 hash. | keyword | -| email.attachments.file.hash.sha256 | SHA256 hash. | keyword | -| email.attachments.file.mime_type | The MIME media type of the attachment. This value will typically be extracted from the `Content-Type` MIME header field. | keyword | -| email.attachments.file.name | Name of the attachment file including the file extension. | keyword | -| email.attachments.file.size | Attachment file size in bytes. | long | -| email.direction | The direction of the message based on the sending and receiving domains. | keyword | -| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | -| email.local_id | Unique identifier given to the email by the source that created the event. Identifier is not persistent across hops. | keyword | -| email.message_id | Identifier from the RFC 5322 `Message-ID:` email header that refers to a particular email message. | wildcard | -| email.subject | A brief summary of the topic of the message. | keyword | -| email.subject.text | Multi-field of `email.subject`. | match_only_text | -| email.to.address | The email address of recipient | keyword | -| error.code | Error code describing the error. | keyword | -| error.message | Error message. | match_only_text | -| error.type | The type of the error, for example the class name of the exception. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.reason | Reason why this event happened, according to the source. This describes the why of a particular action or outcome captured in the event. Where `event.action` captures the action from the event, `event.reason` describes why that action was taken. For example, a web proxy with an `event.action` which denied the request may also populate `event.reason` with the reason why (e.g. `blocked site`). | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | mimecast.AttCnt | The number of attachments on the email. | long | @@ -609,26 +425,6 @@ An example event for `siem` looks as following: | mimecast.log_type | String to get type of SIEM log. | keyword | | mimecast.msgid | The internet message id of the email. | keyword | | mimecast.urlCategory | The category of the URL that was clicked. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| tls.cipher | String indicating the cipher used during the current connection. | keyword | -| tls.established | Boolean flag indicating if the TLS negotiation was successful and transitioned to an encrypted tunnel. | boolean | -| tls.version | Numeric part of the version parsed from the original string. | keyword | -| url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| url.full.text | Multi-field of `url.full`. | match_only_text | -| user.email | User email address. | keyword | ### Threat Intel Feed Malware: Customer @@ -719,50 +515,17 @@ An example event for `threat_intel_malware_customer` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | mimecast.created | When the indicator was last created. | date | | mimecast.hashtype | The hash type. | keyword | | mimecast.id | The ID of the indicator. | keyword | @@ -777,14 +540,6 @@ An example event for `threat_intel_malware_customer` looks as following: | mimecast.type | The indicator type, can for example be "domain, email, FileHash-SHA256". | keyword | | mimecast.valid_from | The valid from date. | date | | mimecast.value | The value of the indicator. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| tags | List of keywords used to tag each event. | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | ### Threat Intel Feed Malware: Grid @@ -875,50 +630,17 @@ An example event for `threat_intel_malware_grid` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | mimecast.created | When the indicator was last created. | date | | mimecast.hashtype | The hash type. | keyword | | mimecast.id | The ID of the indicator. | keyword | @@ -933,14 +655,6 @@ An example event for `threat_intel_malware_grid` looks as following: | mimecast.type | The indicator type, can for example be "domain, email, FileHash-SHA256". | keyword | | mimecast.valid_from | The valid from date. | date | | mimecast.value | The value of the indicator. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| tags | List of keywords used to tag each event. | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | ### TTP Attachment Logs @@ -1039,55 +753,15 @@ An example event for `ttp_ap` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.attachments.file.extension | Attachment file extension, excluding the leading dot. | keyword | -| email.attachments.file.hash.sha256 | SHA256 hash. | keyword | -| email.attachments.file.mime_type | The MIME media type of the attachment. This value will typically be extracted from the `Content-Type` MIME header field. | keyword | -| email.attachments.file.name | Name of the attachment file including the file extension. | keyword | -| email.direction | The direction of the message based on the sending and receiving domains. | keyword | -| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | -| email.message_id | Identifier from the RFC 5322 `Message-ID:` email header that refers to a particular email message. | wildcard | -| email.subject | A brief summary of the topic of the message. | keyword | -| email.subject.text | Multi-field of `email.subject`. | match_only_text | -| email.to.address | The email address of recipient | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | mimecast.actionTriggered | The action triggered for the attachment. | keyword | @@ -1102,9 +776,6 @@ An example event for `ttp_ap` looks as following: | mimecast.route | The route of the original email containing the attachment, either - inbound, outbound, internal, or external. | keyword | | mimecast.senderAddress | The sender of the attachment. | keyword | | mimecast.subject | The subject of the email. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | ### TTP Impersonation Logs @@ -1205,51 +876,15 @@ An example event for `ttp_ip` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | -| email.message_id | Identifier from the RFC 5322 `Message-ID:` email header that refers to a particular email message. | wildcard | -| email.subject | A brief summary of the topic of the message. | keyword | -| email.subject.text | Multi-field of `email.subject`. | match_only_text | -| email.to.address | The email address of recipient | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | mimecast.action | The action triggered by the email. | keyword | @@ -1268,11 +903,6 @@ An example event for `ttp_ip` looks as following: | mimecast.subject | The subject of the email. | keyword | | mimecast.taggedExternal | Whether the message was tagged as coming from an external address. | boolean | | mimecast.taggedMalicious | Whether the message was tagged as malicious. | boolean | -| related.ip | All of the IPs seen on your event. | ip | -| rule.name | The name of the rule or signature generating the event. | keyword | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | ### TTP URL Logs @@ -1386,51 +1016,15 @@ An example event for `ttp_url` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.direction | The direction of the message based on the sending and receiving domains. | keyword | -| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | -| email.message_id | Identifier from the RFC 5322 `Message-ID:` email header that refers to a particular email message. | wildcard | -| email.subject | A brief summary of the topic of the message. | keyword | -| email.subject.text | Multi-field of `email.subject`. | match_only_text | -| email.to.address | The email address of recipient | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | mimecast.action | The action that was taken for the click. | keyword | @@ -1450,12 +1044,4 @@ An example event for `ttp_url` looks as following: | mimecast.userAwarenessAction | The action taken by the user if user awareness was applied. | keyword | | mimecast.userEmailAddress | The email address of the user who clicked the link. | keyword | | mimecast.userOverride | The action requested by the user. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| user.email | User email address. | keyword | diff --git a/packages/mimecast/manifest.yml b/packages/mimecast/manifest.yml index 477de3c60a4..a2b3ade0cf7 100644 --- a/packages/mimecast/manifest.yml +++ b/packages/mimecast/manifest.yml @@ -1,13 +1,13 @@ format_version: "3.0.2" name: mimecast title: "Mimecast" -version: "1.25.0" +version: "1.26.0" description: Collect logs from Mimecast with Elastic Agent. type: integration categories: ["security", "email_security"] conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" screenshots: - src: /img/mimecast.png title: Sample screenshot From e7694ca48112f3d6af8708fe6aabfa1640bb68ac Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:09 +0930 Subject: [PATCH 068/121] [netskope] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.7.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/netskope --- packages/netskope/changelog.yml | 5 + .../_dev/test/pipeline/test-common-config.yml | 1 - .../data_stream/alerts/fields/agent.yml | 127 ------------- .../data_stream/alerts/fields/ecs.yml | 106 ----------- .../_dev/test/pipeline/test-common-config.yml | 1 - .../data_stream/events/fields/agent.yml | 140 -------------- .../data_stream/events/fields/ecs.yml | 128 ------------- packages/netskope/docs/README.md | 178 ------------------ packages/netskope/manifest.yml | 4 +- 9 files changed, 7 insertions(+), 683 deletions(-) delete mode 100644 packages/netskope/data_stream/alerts/fields/ecs.yml delete mode 100644 packages/netskope/data_stream/events/fields/ecs.yml diff --git a/packages/netskope/changelog.yml b/packages/netskope/changelog.yml index 47a75faeb2c..b5c411fbc38 100644 --- a/packages/netskope/changelog.yml +++ b/packages/netskope/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.19.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.18.0" changes: - description: Added support for custom TCP options. diff --git a/packages/netskope/data_stream/alerts/_dev/test/pipeline/test-common-config.yml b/packages/netskope/data_stream/alerts/_dev/test/pipeline/test-common-config.yml index 268aa7f67e6..ed9ac911387 100644 --- a/packages/netskope/data_stream/alerts/_dev/test/pipeline/test-common-config.yml +++ b/packages/netskope/data_stream/alerts/_dev/test/pipeline/test-common-config.yml @@ -1,7 +1,6 @@ fields: tags: - preserve_original_event - dynamic_fields: # This can be removed after ES 8.14 is the minimum version. # Relates: https://github.com/elastic/elasticsearch/pull/105689 diff --git a/packages/netskope/data_stream/alerts/fields/agent.yml b/packages/netskope/data_stream/alerts/fields/agent.yml index ee375d8ef36..894e6f12be2 100644 --- a/packages/netskope/data_stream/alerts/fields/agent.yml +++ b/packages/netskope/data_stream/alerts/fields/agent.yml @@ -5,142 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - external: ecs - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - external: ecs - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - external: ecs - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - external: ecs - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/netskope/data_stream/alerts/fields/ecs.yml b/packages/netskope/data_stream/alerts/fields/ecs.yml deleted file mode 100644 index 354597532a7..00000000000 --- a/packages/netskope/data_stream/alerts/fields/ecs.yml +++ /dev/null @@ -1,106 +0,0 @@ -- external: ecs - name: client.bytes -- external: ecs - name: client.port -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.service.name -- external: ecs - name: destination.address -- external: ecs - name: destination.domain -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.geo.postal_code -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.geo.timezone -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: event.id -- external: ecs - name: ecs.version -- external: ecs - name: file.hash.md5 -- external: ecs - name: file.mime_type -- external: ecs - name: file.name -- external: ecs - name: file.path -- external: ecs - name: file.size -- external: ecs - name: http.request.referrer -- external: ecs - name: network.protocol -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: source.address -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.postal_code -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.geo.timezone -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: tags -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: user.email -- external: ecs - name: user.group.name -- external: ecs - name: user.name -- external: ecs - name: user.roles -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version diff --git a/packages/netskope/data_stream/events/_dev/test/pipeline/test-common-config.yml b/packages/netskope/data_stream/events/_dev/test/pipeline/test-common-config.yml index 7175bb1a704..f88bd98f400 100644 --- a/packages/netskope/data_stream/events/_dev/test/pipeline/test-common-config.yml +++ b/packages/netskope/data_stream/events/_dev/test/pipeline/test-common-config.yml @@ -1,7 +1,6 @@ fields: tags: - preserve_original_event - dynamic_fields: # This can be removed after ES 8.14 is the minimum version. # Relates: https://github.com/elastic/elasticsearch/pull/105689 diff --git a/packages/netskope/data_stream/events/fields/agent.yml b/packages/netskope/data_stream/events/fields/agent.yml index f1d064df00a..894e6f12be2 100644 --- a/packages/netskope/data_stream/events/fields/agent.yml +++ b/packages/netskope/data_stream/events/fields/agent.yml @@ -5,155 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - external: ecs - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - external: ecs - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/netskope/data_stream/events/fields/ecs.yml b/packages/netskope/data_stream/events/fields/ecs.yml deleted file mode 100644 index ec27c72f9be..00000000000 --- a/packages/netskope/data_stream/events/fields/ecs.yml +++ /dev/null @@ -1,128 +0,0 @@ -- external: ecs - name: client.bytes -- external: ecs - name: client.nat.ip -- external: ecs - name: client.packets -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.service.name -- external: ecs - name: destination.address -- external: ecs - name: destination.domain -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.geo.name -- external: ecs - name: destination.geo.postal_code -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.geo.timezone -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: ecs.version -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.type -- external: ecs - name: file.hash.md5 -- external: ecs - name: file.mime_type -- external: ecs - name: file.name -- external: ecs - name: file.path -- external: ecs - name: file.size -- external: ecs - name: network.protocol -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: rule.id -- external: ecs - name: rule.name -- external: ecs - name: server.bytes -- external: ecs - name: server.packets -- external: ecs - name: source.address -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.postal_code -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.geo.timezone -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: tags -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: user.email -- external: ecs - name: user.group.name -- external: ecs - name: user.name -- external: ecs - name: user.roles -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version diff --git a/packages/netskope/docs/README.md b/packages/netskope/docs/README.md index 58eba436dfa..dfcb3b29978 100644 --- a/packages/netskope/docs/README.md +++ b/packages/netskope/docs/README.md @@ -56,67 +56,15 @@ Default port: _9021_ | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| client.bytes | Bytes sent from the client to the server. | long | -| client.port | Port of the client. | long | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| cloud.service.name | The cloud service name is intended to distinguish services running on different platforms within a provider, eg AWS EC2 vs Lambda, GCP GCE vs App Engine, Azure VM vs App Server. Examples: app engine, app service, cloud run, fargate, lambda. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.postal_code | Postal code associated with the location. Values appropriate for this field may also be known as a postcode or ZIP code and will vary widely from country to country. | keyword | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | event.dataset | Event dataset | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | | event.module | Event module | constant_keyword | -| file.hash.md5 | MD5 hash. | keyword | -| file.mime_type | MIME type should identify the format of the file or stream of bytes using https://www.iana.org/assignments/media-types/media-types.xhtml[IANA official types], where possible. When more than one type is applicable, the most specific type should be used. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.referrer | Referrer for this HTTP request. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -544,37 +492,6 @@ Default port: _9021_ | netskope.alerts.workspace.id | Workspace ID in case of Slack for Enterprise. | keyword | | netskope.alerts.workspace.name | Workspace name in case of Slack for Enterprise. | keyword | | netskope.alerts.zip.password | Zip the malicious file and put pwd to it and send it back to caller. | keyword | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.postal_code | Postal code associated with the location. Values appropriate for this field may also be known as a postcode or ZIP code and will vary widely from country to country. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| user.email | User email address. | keyword | -| user.group.name | Name of the group. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user.roles | Array of user roles at the time of the event. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | An example event for `alerts` looks as following: @@ -769,72 +686,15 @@ An example event for `alerts` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| client.bytes | Bytes sent from the client to the server. | long | -| client.nat.ip | Translated IP of source based NAT sessions (e.g. internal client to internet). Typically connections traversing load balancers, firewalls, or routers. | ip | -| client.packets | Packets sent from the client to the server. | long | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| cloud.service.name | The cloud service name is intended to distinguish services running on different platforms within a provider, eg AWS EC2 vs Lambda, GCP GCE vs App Engine, Azure VM vs App Server. Examples: app engine, app service, cloud run, fargate, lambda. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| destination.geo.postal_code | Postal code associated with the location. Values appropriate for this field may also be known as a postcode or ZIP code and will vary widely from country to country. | keyword | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | | event.dataset | Event dataset | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.hash.md5 | MD5 hash. | keyword | -| file.mime_type | MIME type should identify the format of the file or stream of bytes using https://www.iana.org/assignments/media-types/media-types.xhtml[IANA official types], where possible. When more than one type is applicable, the most specific type should be used. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | @@ -1134,44 +994,6 @@ An example event for `alerts` looks as following: | netskope.events.workspace.id | Workspace ID in case of Slack for Enterprise. | keyword | | netskope.events.workspace.name | Workspace name in case of Slack for Enterprise. | keyword | | netskope.events.zip_password | Zip the malacious file and put pwd to it and send it back to caller. | keyword | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| server.bytes | Bytes sent from the server to the client. | long | -| server.packets | Packets sent from the server to the client. | long | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.postal_code | Postal code associated with the location. Values appropriate for this field may also be known as a postcode or ZIP code and will vary widely from country to country. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| user.email | User email address. | keyword | -| user.group.name | Name of the group. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user.roles | Array of user roles at the time of the event. | keyword | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | An example event for `events` looks as following: diff --git a/packages/netskope/manifest.yml b/packages/netskope/manifest.yml index 1a11db082cb..284b4c7e8a8 100644 --- a/packages/netskope/manifest.yml +++ b/packages/netskope/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: netskope title: "Netskope" -version: "1.18.0" +version: "1.19.0" description: Collect logs from Netskope with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - network conditions: kibana: - version: ^8.7.0 + version: "^8.13.0" screenshots: - src: /img/netskope-alerts-screenshot.png title: Netskope Alert logs screenshot From a9389db2896406c448e2d79684ce75b0a16aef4a Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:19 +0930 Subject: [PATCH 069/121] [o365] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/o365 --- packages/o365/changelog.yml | 5 + .../o365/data_stream/audit/fields/agent.yml | 152 +---------------- .../o365/data_stream/audit/fields/beats.yml | 3 - .../o365/data_stream/audit/fields/ecs.yml | 156 ------------------ packages/o365/docs/README.md | 116 ------------- packages/o365/manifest.yml | 4 +- 6 files changed, 8 insertions(+), 428 deletions(-) delete mode 100644 packages/o365/data_stream/audit/fields/ecs.yml diff --git a/packages/o365/changelog.yml b/packages/o365/changelog.yml index 9977d04563b..f3e4dfe3485 100644 --- a/packages/o365/changelog.yml +++ b/packages/o365/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.4.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.3.3" changes: - description: Improve handling of o365.audit.AdditionalInfo. diff --git a/packages/o365/data_stream/audit/fields/agent.yml b/packages/o365/data_stream/audit/fields/agent.yml index 92d4ec0730c..2bc58530bac 100644 --- a/packages/o365/data_stream/audit/fields/agent.yml +++ b/packages/o365/data_stream/audit/fields/agent.yml @@ -5,165 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - external: ecs - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - external: ecs - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - external: ecs - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/o365/data_stream/audit/fields/beats.yml b/packages/o365/data_stream/audit/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/o365/data_stream/audit/fields/beats.yml +++ b/packages/o365/data_stream/audit/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/o365/data_stream/audit/fields/ecs.yml b/packages/o365/data_stream/audit/fields/ecs.yml deleted file mode 100644 index 91fa0a69b67..00000000000 --- a/packages/o365/data_stream/audit/fields/ecs.yml +++ /dev/null @@ -1,156 +0,0 @@ -- external: ecs - name: client.address -- external: ecs - name: client.domain -- external: ecs - name: client.ip -- external: ecs - name: client.port -- external: ecs - name: destination.ip -- external: ecs - name: destination.user.email -- external: ecs - name: destination.user.id -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.code -- external: ecs - name: event.id -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.outcome -- external: ecs - name: event.provider -- external: ecs - name: event.severity -- external: ecs - name: event.type -- external: ecs - name: file.directory -- external: ecs - name: file.extension -- external: ecs - name: file.inode -- external: ecs - name: file.mtime -- external: ecs - name: file.name -- external: ecs - name: file.owner -- external: ecs - name: group.name -- external: ecs - name: message -- external: ecs - name: network.type -- external: ecs - name: organization.id -- external: ecs - name: organization.name -- external: ecs - name: process.name -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: rule.category -- external: ecs - name: rule.description -- external: ecs - name: rule.id -- external: ecs - name: rule.name -- external: ecs - name: rule.reference -- external: ecs - name: rule.ruleset -- external: ecs - name: server.address -- external: ecs - name: server.domain -- external: ecs - name: server.ip -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.name -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: source.user.email -- external: ecs - name: tags -- external: ecs - name: threat.technique.id -- external: ecs - name: url.original -- external: ecs - name: user.domain -- external: ecs - name: user.email -- external: ecs - name: user.full_name -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user.target.domain -- external: ecs - name: user.target.email -- external: ecs - name: user.target.full_name -- external: ecs - name: user.target.group.domain -- external: ecs - name: user.target.group.id -- external: ecs - name: user.target.group.name -- external: ecs - name: user.target.id -- external: ecs - name: user.target.name -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version diff --git a/packages/o365/docs/README.md b/packages/o365/docs/README.md index af1bc05c8c8..0024af294a7 100644 --- a/packages/o365/docs/README.md +++ b/packages/o365/docs/README.md @@ -178,73 +178,18 @@ An example event for `audit` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| client.address | Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| client.domain | The domain name of the client system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| client.port | Port of the client. | long | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.user.email | User email address. | keyword | -| destination.user.id | Unique identifier of the user. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | | event.dataset | Event dataset | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.directory | Directory where the file is located. It should include the drive letter, when appropriate. | keyword | -| file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| file.inode | Inode representing the file in the filesystem. | keyword | -| file.mtime | Last time the file content was modified. | date | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.owner | File owner's username. | keyword | -| group.name | Name of the group. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | | o365.audit.Activity | | keyword | | o365.audit.Actor.ID | | keyword | | o365.audit.Actor.Type | | keyword | @@ -415,65 +360,4 @@ An example event for `audit` looks as following: | o365.audit.WorkspaceId | | keyword | | o365.audit.WorkspaceName | | keyword | | o365.audit.YammerNetworkId | | keyword | -| organization.id | Unique identifier for the organization. | keyword | -| organization.name | Organization name. | keyword | -| organization.name.text | Multi-field of `organization.name`. | match_only_text | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.category | A categorization value keyword used by the entity using the rule for detection of this event. | keyword | -| rule.description | The description of the rule generating the event. | keyword | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| rule.reference | Reference URL to additional information about the rule used to generate this event. The URL can point to the vendor's documentation about the rule. If that's not available, it can also be a link to a more general page describing this type of alert. | keyword | -| rule.ruleset | Name of the ruleset, policy, group, or parent category in which the rule used to generate this event is a member. | keyword | -| server.address | Some event server addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| server.domain | The domain name of the server system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| server.ip | IP address of the server (IPv4 or IPv6). | ip | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| source.user.email | User email address. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| threat.technique.id | The id of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.email | User email address. | keyword | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user.target.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.target.email | User email address. | keyword | -| user.target.full_name | User's full name, if available. | keyword | -| user.target.full_name.text | Multi-field of `user.target.full_name`. | match_only_text | -| user.target.group.domain | Name of the directory the group is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.target.group.id | Unique identifier for the group on the system/platform. | keyword | -| user.target.group.name | Name of the group. | keyword | -| user.target.id | Unique identifier of the user. | keyword | -| user.target.name | Short name or login of the user. | keyword | -| user.target.name.text | Multi-field of `user.target.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | diff --git a/packages/o365/manifest.yml b/packages/o365/manifest.yml index fe7d562be53..0d3d815c6f2 100644 --- a/packages/o365/manifest.yml +++ b/packages/o365/manifest.yml @@ -1,13 +1,13 @@ name: o365 title: Microsoft 365 -version: "2.3.3" +version: "2.4.0" description: Collect logs from Microsoft 365 with Elastic Agent. type: integration format_version: "3.0.2" categories: [security, productivity_security] conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" icons: - src: /img/logo-integrations-microsoft-365.svg title: Microsoft Office 365 From 9ebcf85d8caa24cd9de628db6085210fc663747e Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:20 +0930 Subject: [PATCH 070/121] [okta] - Updated fields definitions Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/okta --- packages/okta/changelog.yml | 5 + .../okta/data_stream/system/fields/agent.yml | 164 +----------------- .../okta/data_stream/system/fields/beats.yml | 3 - .../okta/data_stream/system/fields/ecs.yml | 144 --------------- packages/okta/docs/README.md | 113 ------------ packages/okta/manifest.yml | 2 +- 6 files changed, 7 insertions(+), 424 deletions(-) delete mode 100644 packages/okta/data_stream/system/fields/ecs.yml diff --git a/packages/okta/changelog.yml b/packages/okta/changelog.yml index c8f7f9a2f13..7fc0e0c447a 100644 --- a/packages/okta/changelog.yml +++ b/packages/okta/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.11.0" + changes: + - description: Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.10.0" changes: - description: Support OIN service application authentication. diff --git a/packages/okta/data_stream/system/fields/agent.yml b/packages/okta/data_stream/system/fields/agent.yml index 2f0666cc38f..2bc58530bac 100644 --- a/packages/okta/data_stream/system/fields/agent.yml +++ b/packages/okta/data_stream/system/fields/agent.yml @@ -5,177 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - external: ecs - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/okta/data_stream/system/fields/beats.yml b/packages/okta/data_stream/system/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/okta/data_stream/system/fields/beats.yml +++ b/packages/okta/data_stream/system/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/okta/data_stream/system/fields/ecs.yml b/packages/okta/data_stream/system/fields/ecs.yml deleted file mode 100644 index 688246885b3..00000000000 --- a/packages/okta/data_stream/system/fields/ecs.yml +++ /dev/null @@ -1,144 +0,0 @@ -- external: ecs - name: client.as.number -- external: ecs - name: client.as.organization.name -- external: ecs - name: client.domain -- external: ecs - name: client.geo.city_name -- external: ecs - name: client.geo.country_name -- external: ecs - name: client.geo.location -- external: ecs - name: client.geo.region_name -- external: ecs - name: client.ip -- external: ecs - name: client.user.full_name -- external: ecs - name: client.user.id -- external: ecs - name: client.user.name -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.geo.name -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.ip -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.type -- external: ecs - name: message -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.domain -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.name -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: source.user.full_name -- external: ecs - name: source.user.id -- external: ecs - name: source.user.name -- external: ecs - name: tags -- external: ecs - name: user.domain -- external: ecs - name: user.email -- external: ecs - name: user.full_name -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user.target.domain -- external: ecs - name: user.target.email -- external: ecs - name: user.target.full_name -- external: ecs - name: user.target.group.domain -- external: ecs - name: user.target.group.id -- external: ecs - name: user.target.group.name -- external: ecs - name: user.target.id -- external: ecs - name: user.target.name -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version diff --git a/packages/okta/docs/README.md b/packages/okta/docs/README.md index 2e5303f958d..4920e98dbbc 100644 --- a/packages/okta/docs/README.md +++ b/packages/okta/docs/README.md @@ -224,83 +224,18 @@ An example event for `system` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| client.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| client.as.organization.name | Organization name. | keyword | -| client.as.organization.name.text | Multi-field of `client.as.organization.name`. | match_only_text | -| client.domain | The domain name of the client system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| client.geo.city_name | City name. | keyword | -| client.geo.country_name | Country name. | keyword | -| client.geo.location | Longitude and latitude. | geo_point | -| client.geo.region_name | Region name. | keyword | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| client.user.full_name | User's full name, if available. | keyword | -| client.user.full_name.text | Multi-field of `client.user.full_name`. | match_only_text | -| client.user.id | Unique identifier of the user. | keyword | -| client.user.name | Short name or login of the user. | keyword | -| client.user.name.text | Multi-field of `client.user.name`. | match_only_text | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | okta.actor.alternate_id | Alternate identifier of the actor. | keyword | | okta.actor.display_name | Display name of the actor. | keyword | | okta.actor.id | Identifier of the actor. | keyword | @@ -358,51 +293,3 @@ An example event for `system` looks as following: | okta.transaction.type | The type of transaction. Must be one of "WEB", "JOB". | keyword | | okta.uuid | The unique identifier of the Okta LogEvent. | keyword | | okta.version | The version of the LogEvent. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.user.full_name | User's full name, if available. | keyword | -| source.user.full_name.text | Multi-field of `source.user.full_name`. | match_only_text | -| source.user.id | Unique identifier of the user. | keyword | -| source.user.name | Short name or login of the user. | keyword | -| source.user.name.text | Multi-field of `source.user.name`. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.email | User email address. | keyword | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user.target.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.target.email | User email address. | keyword | -| user.target.full_name | User's full name, if available. | keyword | -| user.target.full_name.text | Multi-field of `user.target.full_name`. | match_only_text | -| user.target.group.domain | Name of the directory the group is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.target.group.id | Unique identifier for the group on the system/platform. | keyword | -| user.target.group.name | Name of the group. | keyword | -| user.target.id | Unique identifier of the user. | keyword | -| user.target.name | Short name or login of the user. | keyword | -| user.target.name.text | Multi-field of `user.target.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | diff --git a/packages/okta/manifest.yml b/packages/okta/manifest.yml index 62f68c3a9f9..8c21b151717 100644 --- a/packages/okta/manifest.yml +++ b/packages/okta/manifest.yml @@ -1,6 +1,6 @@ name: okta title: Okta -version: "2.10.0" +version: "2.11.0" description: Collect and parse event logs from Okta API with Elastic Agent. type: integration format_version: "3.0.2" From 55ec1154be405a6724cf4521d21a7a5d4bf93c29 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:21 +0930 Subject: [PATCH 071/121] [opencanary] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.2 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/opencanary --- packages/opencanary/changelog.yml | 5 + .../pipeline/test-events.log-expected.json | 2 +- .../data_stream/events/fields/agent.yml | 167 +-------- .../data_stream/events/fields/ecs.yml | 327 ------------------ packages/opencanary/docs/README.md | 203 ----------- packages/opencanary/manifest.yml | 4 +- 6 files changed, 9 insertions(+), 699 deletions(-) delete mode 100644 packages/opencanary/data_stream/events/fields/ecs.yml diff --git a/packages/opencanary/changelog.yml b/packages/opencanary/changelog.yml index 6bbdef39abc..e780a4d12d5 100644 --- a/packages/opencanary/changelog.yml +++ b/packages/opencanary/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.1.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "0.0.1" changes: - description: Initial draft of the package diff --git a/packages/opencanary/data_stream/events/_dev/test/pipeline/test-events.log-expected.json b/packages/opencanary/data_stream/events/_dev/test/pipeline/test-events.log-expected.json index 49aacae58ac..7a0057553e9 100644 --- a/packages/opencanary/data_stream/events/_dev/test/pipeline/test-events.log-expected.json +++ b/packages/opencanary/data_stream/events/_dev/test/pipeline/test-events.log-expected.json @@ -502,7 +502,7 @@ "id": "opencanary-1" }, "redis": { - "command": "\u0000\u000c\u0000\u0000\u0010\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" + "command": "\u0000\f\u0000\u0000\u0010\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000" } }, "related": { diff --git a/packages/opencanary/data_stream/events/fields/agent.yml b/packages/opencanary/data_stream/events/fields/agent.yml index 060a12cbb09..8f9dc95f3a3 100644 --- a/packages/opencanary/data_stream/events/fields/agent.yml +++ b/packages/opencanary/data_stream/events/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/opencanary/data_stream/events/fields/ecs.yml b/packages/opencanary/data_stream/events/fields/ecs.yml deleted file mode 100644 index 96cfa7d0df8..00000000000 --- a/packages/opencanary/data_stream/events/fields/ecs.yml +++ /dev/null @@ -1,327 +0,0 @@ -- external: ecs - name: client.domain -- external: ecs - name: client.address -- external: ecs - name: client.port -- external: ecs - name: client.ip -- external: ecs - name: client.user.name -- external: ecs - name: destination.address -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.bytes -- external: ecs - name: destination.domain -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_code -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.geo.postal_code -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.geo.timezone -- external: ecs - name: destination.ip -- external: ecs - name: destination.mac -- external: ecs - name: destination.nat.ip -- external: ecs - name: destination.nat.port -- external: ecs - name: destination.packets -- external: ecs - name: destination.port -- external: ecs - name: destination.user.name -- external: ecs - name: dns.question.name -- external: ecs - name: dns.question.registered_domain -- external: ecs - name: dns.question.subdomain -- external: ecs - name: dns.question.top_level_domain -- external: ecs - name: dns.question.type -- external: ecs - name: dns.response_code -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.code -- external: ecs - name: event.created -- external: ecs - name: event.duration -- external: ecs - name: event.end -- external: ecs - name: event.id -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.provider -- external: ecs - name: event.reason -- external: ecs - name: event.severity -- external: ecs - name: event.start -- external: ecs - name: event.timezone -- external: ecs - name: event.type -- external: ecs - name: file.hash.sha256 -- external: ecs - name: file.name -- external: ecs - name: file.path -- external: ecs - name: file.size -- external: ecs - name: http.request.method -- external: ecs - name: http.request.bytes -- external: ecs - name: http.request.referrer -- external: ecs - name: http.response.bytes -- external: ecs - name: http.response.status_code -- external: ecs - name: labels -- external: ecs - name: log.file.path -- external: ecs - name: log.level -- external: ecs - name: log.logger -- external: ecs - name: log.syslog.priority -- external: ecs - name: log.syslog.facility.code -- external: ecs - name: log.syslog.facility.name -- external: ecs - name: log.syslog.hostname -- external: ecs - name: log.syslog.severity.code -- external: ecs - name: log.syslog.severity.name -- external: ecs - name: message -- external: ecs - name: network.application -- external: ecs - name: network.bytes -- external: ecs - name: network.community_id -- external: ecs - name: network.direction -- external: ecs - name: network.iana_number -- external: ecs - name: network.inner - type: group -- external: ecs - name: network.inner.vlan.id -- external: ecs - name: network.inner.vlan.name -- external: ecs - name: network.protocol -- external: ecs - name: network.transport -- external: ecs - name: network.type -- external: ecs - name: observer.egress.interface.alias -- external: ecs - name: observer.egress.interface.id -- external: ecs - name: observer.egress.interface.name -- external: ecs - name: observer.egress.zone -- external: ecs - name: observer.hostname -- external: ecs - name: observer.ingress.interface.alias -- external: ecs - name: observer.ingress.interface.id -- external: ecs - name: observer.ingress.interface.name -- external: ecs - name: observer.ingress.zone -- external: ecs - name: observer.ip -- external: ecs - name: observer.name -- external: ecs - name: observer.product -- external: ecs - name: observer.type -- external: ecs - name: observer.vendor -- external: ecs - name: observer.version -- external: ecs - name: process.name -- external: ecs - name: process.pid -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: rule.category -- external: ecs - name: rule.id -- external: ecs - name: rule.name -- external: ecs - name: rule.ruleset -- external: ecs - name: server.domain -- external: ecs - name: server.address -- external: ecs - name: server.port -- external: ecs - name: server.ip -- external: ecs - name: server.user.name -- external: ecs - name: service.id -- external: ecs - name: source.address -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.bytes -- external: ecs - name: source.domain -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_code -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.postal_code -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.geo.timezone -- external: ecs - name: source.ip -- external: ecs - name: source.mac -- external: ecs - name: source.nat.ip -- external: ecs - name: source.nat.port -- external: ecs - name: source.packets -- external: ecs - name: source.port -- external: ecs - name: source.user.name -- external: ecs - name: source.user.group.name -- external: ecs - name: tags -- external: ecs - name: url.domain -- external: ecs - name: url.extension -- external: ecs - name: url.fragment -- external: ecs - name: url.full -- external: ecs - name: url.original -- external: ecs - name: url.password -- external: ecs - name: url.path -- external: ecs - name: url.port -- external: ecs - name: url.query -- external: ecs - name: url.registered_domain -- external: ecs - name: url.scheme -- external: ecs - name: url.subdomain -- external: ecs - name: url.top_level_domain -- external: ecs - name: url.username -- external: ecs - name: user.domain -- external: ecs - name: user.email -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version diff --git a/packages/opencanary/docs/README.md b/packages/opencanary/docs/README.md index a4f8f58ebe2..1a09eb7957b 100755 --- a/packages/opencanary/docs/README.md +++ b/packages/opencanary/docs/README.md @@ -89,145 +89,17 @@ An example event for `events` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | -| client.address | Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| client.domain | The domain name of the client system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| client.port | Port of the client. | long | -| client.user.name | Short name or login of the user. | keyword | -| client.user.name.text | Multi-field of `client.user.name`. | match_only_text | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.bytes | Bytes sent from the destination to the source. | long | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_code | Two-letter code representing continent's name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.postal_code | Postal code associated with the location. Values appropriate for this field may also be known as a postcode or ZIP code and will vary widely from country to country. | keyword | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.mac | MAC address of the destination. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| destination.nat.ip | Translated ip of destination based NAT sessions (e.g. internet to private DMZ) Typically used with load balancers, firewalls, or routers. | ip | -| destination.nat.port | Port the source session is translated to by NAT Device. Typically used with load balancers, firewalls, or routers. | long | -| destination.packets | Packets sent from the destination to the source. | long | -| destination.port | Port of the destination. | long | -| destination.user.name | Short name or login of the user. | keyword | -| destination.user.name.text | Multi-field of `destination.user.name`. | match_only_text | -| dns.question.name | The name being queried. If the name field contains non-printable characters (below 32 or above 126), those characters should be represented as escaped base 10 integers (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, and line feeds should be converted to \t, \r, and \n respectively. | keyword | -| dns.question.registered_domain | The highest registered domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| dns.question.subdomain | The subdomain is all of the labels under the registered_domain. If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| dns.question.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| dns.question.type | The type of record being queried. | keyword | -| dns.response_code | The DNS response code. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.duration | Duration of the event in nanoseconds. If `event.start` and `event.end` are known this value should be the difference between the end and start time. | long | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.reason | Reason why this event happened, according to the source. This describes the why of a particular action or outcome captured in the event. Where `event.action` captures the action from the event, `event.reason` describes why that action was taken. For example, a web proxy with an `event.action` which denied the request may also populate `event.reason` with the reason why (e.g. `blocked site`). | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.bytes | Total size in bytes of the request (body and headers). | long | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.request.referrer | Referrer for this HTTP request. | keyword | -| http.response.bytes | Total size in bytes of the response (body and headers). | long | -| http.response.status_code | HTTP response status code. | long | | input.type | Input type. | keyword | -| labels | Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels. | object | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | -| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | -| log.logger | The name of the logger inside an application. This is usually the name of the class which initialized the logger, or can be a custom name. | keyword | | log.offset | Offset of the entry in the log file. | long | -| log.syslog.facility.code | The Syslog numeric facility of the log event, if available. According to RFCs 5424 and 3164, this value should be an integer between 0 and 23. | long | -| log.syslog.facility.name | The Syslog text-based facility of the log event, if available. | keyword | -| log.syslog.hostname | The hostname, FQDN, or IP of the machine that originally sent the Syslog message. This is sourced from the hostname field of the syslog header. Depending on the environment, this value may be different from the host that handled the event, especially if the host handling the events is acting as a collector. | keyword | -| log.syslog.priority | Syslog numeric priority of the event, if available. According to RFCs 5424 and 3164, the priority is 8 \* facility + severity. This number is therefore expected to contain a value between 0 and 191. | long | -| log.syslog.severity.code | The Syslog numeric severity of the log event, if available. If the event source publishing via Syslog provides a different numeric severity value (e.g. firewall, IDS), your source's numeric severity should go to `event.severity`. If the event source does not specify a distinct severity, you can optionally copy the Syslog severity to `event.severity`. | long | -| log.syslog.severity.name | The Syslog numeric severity of the log event, if available. If the event source publishing via Syslog provides a different severity value (e.g. firewall, IDS), your source's text severity should go to `log.level`. If the event source does not specify a distinct severity, you can optionally copy the Syslog severity to `log.level`. | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.application | When a specific application or service is identified from network connection details (source/dest IPs, ports, certificates, or wire format), this field captures the application's or service's name. For example, the original event identifies the network connection being from a specific web service in a `https` network connection, like `facebook` or `twitter`. The field value must be normalized to lowercase for querying. | keyword | -| network.bytes | Total bytes transferred in both directions. If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. | long | -| network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword | -| network.inner | Network.inner fields are added in addition to network.vlan fields to describe the innermost VLAN when q-in-q VLAN tagging is present. Allowed fields include vlan.id and vlan.name. Inner vlan fields are typically used when sending traffic with multiple 802.1q encapsulations to a network sensor (e.g. Zeek, Wireshark.) | group | -| network.inner.vlan.id | VLAN ID as reported by the observer. | keyword | -| network.inner.vlan.name | Optional VLAN name as reported by the observer. | keyword | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | -| observer.egress.interface.alias | Interface alias as reported by the system, typically used in firewall implementations for e.g. inside, outside, or dmz logical interface naming. | keyword | -| observer.egress.interface.id | Interface ID as reported by an observer (typically SNMP interface ID). | keyword | -| observer.egress.interface.name | Interface name as reported by the system. | keyword | -| observer.egress.zone | Network zone of outbound traffic as reported by the observer to categorize the destination area of egress traffic, e.g. Internal, External, DMZ, HR, Legal, etc. | keyword | -| observer.hostname | Hostname of the observer. | keyword | -| observer.ingress.interface.alias | Interface alias as reported by the system, typically used in firewall implementations for e.g. inside, outside, or dmz logical interface naming. | keyword | -| observer.ingress.interface.id | Interface ID as reported by an observer (typically SNMP interface ID). | keyword | -| observer.ingress.interface.name | Interface name as reported by the system. | keyword | -| observer.ingress.zone | Network zone of incoming traffic as reported by the observer to categorize the source area of ingress traffic. e.g. internal, External, DMZ, HR, Legal, etc. | keyword | -| observer.ip | IP addresses of the observer. | ip | -| observer.name | Custom name of the observer. This is a name that can be given to an observer. This can be helpful for example if multiple firewalls of the same model are used in an organization. If no custom name is needed, the field can be left empty. | keyword | -| observer.product | The product name of the observer. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| observer.version | Observer version. | keyword | | opencanary.logdata.cwr | | keyword | | opencanary.logdata.df | | keyword | | opencanary.logdata.ece | | keyword | @@ -261,79 +133,4 @@ An example event for `events` looks as following: | opencanary.tcp_banner.data | | keyword | | opencanary.tcp_banner.function | | keyword | | opencanary.tcp_banner.secret_string | | keyword | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.pid | Process id. | long | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.category | A categorization value keyword used by the entity using the rule for detection of this event. | keyword | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| rule.ruleset | Name of the ruleset, policy, group, or parent category in which the rule used to generate this event is a member. | keyword | -| server.address | Some event server addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| server.domain | The domain name of the server system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| server.ip | IP address of the server (IPv4 or IPv6). | ip | -| server.port | Port of the server. | long | -| server.user.name | Short name or login of the user. | keyword | -| server.user.name.text | Multi-field of `server.user.name`. | match_only_text | -| service.id | Unique identifier of the running service. If the service is comprised of many nodes, the `service.id` should be the same for all nodes. This id should uniquely identify the service. This makes it possible to correlate logs and metrics for one specific service, no matter which particular node emitted the event. Note that if you need to see the events from one specific host of the service, you should filter on that `host.name` or `host.id` instead. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_code | Two-letter code representing continent's name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.postal_code | Postal code associated with the location. Values appropriate for this field may also be known as a postcode or ZIP code and will vary widely from country to country. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.mac | MAC address of the source. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| source.nat.ip | Translated ip of source based NAT sessions (e.g. internal client to internet) Typically connections traversing load balancers, firewalls, or routers. | ip | -| source.nat.port | Translated port of source based NAT sessions. (e.g. internal client to internet) Typically used with load balancers, firewalls, or routers. | long | -| source.packets | Packets sent from the source to the destination. | long | -| source.port | Port of the source. | long | -| source.user.group.name | Name of the group. | keyword | -| source.user.name | Short name or login of the user. | keyword | -| source.user.name.text | Multi-field of `source.user.name`. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | -| url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| url.full.text | Multi-field of `url.full`. | match_only_text | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.password | Password of the request. | keyword | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| url.username | Username of the request. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | diff --git a/packages/opencanary/manifest.yml b/packages/opencanary/manifest.yml index 61dc41b1314..44bc0d76079 100644 --- a/packages/opencanary/manifest.yml +++ b/packages/opencanary/manifest.yml @@ -1,14 +1,14 @@ format_version: 3.1.3 name: opencanary title: "OpenCanary" -version: 0.0.1 +version: "0.1.0" description: "This integration collects and parses logs from OpenCanary honeypots." type: integration categories: - security conditions: kibana: - version: "^8.12.2" + version: "^8.13.0" elastic: subscription: "basic" icons: From 43d0b20226c118b5459be7c6858e2a17ff01be7d Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:23 +0930 Subject: [PATCH 072/121] [panw_cortex_xdr] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/panw_cortex_xdr --- packages/panw_cortex_xdr/changelog.yml | 5 + .../data_stream/alerts/fields/agent.yml | 139 +-------------- .../data_stream/alerts/fields/beats.yml | 3 - .../data_stream/alerts/fields/ecs.yml | 156 ----------------- .../data_stream/incidents/fields/agent.yml | 139 +-------------- .../data_stream/incidents/fields/beats.yml | 3 - .../data_stream/incidents/fields/ecs.yml | 32 ---- packages/panw_cortex_xdr/docs/README.md | 162 ------------------ packages/panw_cortex_xdr/manifest.yml | 4 +- 9 files changed, 9 insertions(+), 634 deletions(-) delete mode 100644 packages/panw_cortex_xdr/data_stream/alerts/fields/ecs.yml delete mode 100644 packages/panw_cortex_xdr/data_stream/incidents/fields/ecs.yml diff --git a/packages/panw_cortex_xdr/changelog.yml b/packages/panw_cortex_xdr/changelog.yml index f23d21d76be..cd54a12094c 100644 --- a/packages/panw_cortex_xdr/changelog.yml +++ b/packages/panw_cortex_xdr/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.27.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.26.0" changes: - description: Improve handling of empty responses. diff --git a/packages/panw_cortex_xdr/data_stream/alerts/fields/agent.yml b/packages/panw_cortex_xdr/data_stream/alerts/fields/agent.yml index fed14316c18..2bc58530bac 100644 --- a/packages/panw_cortex_xdr/data_stream/alerts/fields/agent.yml +++ b/packages/panw_cortex_xdr/data_stream/alerts/fields/agent.yml @@ -5,152 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - external: ecs - - name: hostname - external: ecs - - name: id - external: ecs - - name: ip - external: ecs - - name: mac - external: ecs - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - external: ecs - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/panw_cortex_xdr/data_stream/alerts/fields/beats.yml b/packages/panw_cortex_xdr/data_stream/alerts/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/panw_cortex_xdr/data_stream/alerts/fields/beats.yml +++ b/packages/panw_cortex_xdr/data_stream/alerts/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/panw_cortex_xdr/data_stream/alerts/fields/ecs.yml b/packages/panw_cortex_xdr/data_stream/alerts/fields/ecs.yml deleted file mode 100644 index 961c6312569..00000000000 --- a/packages/panw_cortex_xdr/data_stream/alerts/fields/ecs.yml +++ /dev/null @@ -1,156 +0,0 @@ -- name: ecs.version - external: ecs -- name: message - external: ecs -- name: tags - external: ecs -- name: email.from.address - external: ecs -- name: email.to.address - external: ecs -- name: email.subject - external: ecs -- name: event.kind - external: ecs -- name: event.original - external: ecs -- name: event.type - external: ecs -- name: event.category - external: ecs -- name: event.ingested - external: ecs -- name: event.created - external: ecs -- name: event.severity - external: ecs -- name: event.action - external: ecs -- name: event.reason - external: ecs -- name: dns.question.name - external: ecs -- name: destination.ip - external: ecs -- name: destination.port - external: ecs -- name: destination.as.number - external: ecs -- name: destination.as.organization.name - external: ecs -- name: destination.geo.continent_name - external: ecs -- name: destination.geo.city_name - external: ecs -- name: destination.geo.country_iso_code - external: ecs -- name: destination.geo.country_name - external: ecs -- name: destination.geo.region_iso_code - external: ecs -- name: destination.geo.region_name - external: ecs -- name: destination.geo.location - external: ecs -- name: source.ip - external: ecs -- name: source.port - external: ecs -- name: source.as.number - external: ecs -- name: source.as.organization.name - external: ecs -- name: source.geo.continent_name - external: ecs -- name: source.geo.country_iso_code - external: ecs -- name: source.geo.country_name - external: ecs -- name: source.geo.location - external: ecs -- name: process.hash.sha256 - external: ecs -- name: process.command_line - external: ecs -- name: process.name - external: ecs -- name: process.code_signature.subject_name - external: ecs -- name: process.code_signature.status - external: ecs -- name: process.entity_id - external: ecs -- name: process.pid - external: ecs -- name: process.executable - external: ecs -- name: process.hash.md5 - external: ecs -- name: process.thread.id - external: ecs -- name: process.parent.name - external: ecs -- name: process.parent.executable - external: ecs -- name: process.parent.hash.md5 - external: ecs -- name: process.parent.hash.sha256 - external: ecs -- name: process.parent.entity_id - external: ecs -- name: process.parent.code_signature.subject_name - external: ecs -- name: process.parent.code_signature.status - external: ecs -- name: process.parent.command_line - external: ecs -- name: process.parent.uptime - external: ecs -- name: file.path - external: ecs -- name: file.name - external: ecs -- name: file.hash.md5 - external: ecs -- name: file.hash.sha256 - external: ecs -- name: user.name - external: ecs -- name: user.domain - external: ecs -- name: user.id - external: ecs -- name: user.email - external: ecs -- name: rule.name - external: ecs -- name: rule.id - external: ecs -- name: observer.ingress.interface.name - external: ecs -- name: observer.egress.interface.name - external: ecs -- name: observer.serial_number - external: ecs -- name: registry.key - external: ecs -- name: registry.value - external: ecs -- name: registry.path - external: ecs -- name: registry.data.strings - external: ecs -- name: related.hash - external: ecs -- name: related.user - external: ecs -- name: threat.framework - external: ecs -- name: threat.technique.id - external: ecs -- name: threat.technique.name - external: ecs -- name: threat.tactic.id - external: ecs -- name: threat.tactic.name - external: ecs diff --git a/packages/panw_cortex_xdr/data_stream/incidents/fields/agent.yml b/packages/panw_cortex_xdr/data_stream/incidents/fields/agent.yml index fed14316c18..2bc58530bac 100644 --- a/packages/panw_cortex_xdr/data_stream/incidents/fields/agent.yml +++ b/packages/panw_cortex_xdr/data_stream/incidents/fields/agent.yml @@ -5,152 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - external: ecs - - name: hostname - external: ecs - - name: id - external: ecs - - name: ip - external: ecs - - name: mac - external: ecs - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - external: ecs - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/panw_cortex_xdr/data_stream/incidents/fields/beats.yml b/packages/panw_cortex_xdr/data_stream/incidents/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/panw_cortex_xdr/data_stream/incidents/fields/beats.yml +++ b/packages/panw_cortex_xdr/data_stream/incidents/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/panw_cortex_xdr/data_stream/incidents/fields/ecs.yml b/packages/panw_cortex_xdr/data_stream/incidents/fields/ecs.yml deleted file mode 100644 index c629e96c1a3..00000000000 --- a/packages/panw_cortex_xdr/data_stream/incidents/fields/ecs.yml +++ /dev/null @@ -1,32 +0,0 @@ -- name: ecs.version - external: ecs -- name: message - external: ecs -- name: tags - external: ecs -- name: user.name - external: ecs -- name: user.domain - external: ecs -- name: user.id - external: ecs -- name: user.email - external: ecs -- name: rule.name - external: ecs -- name: rule.id - external: ecs -- name: related.hosts - external: ecs -- name: related.user - external: ecs -- name: threat.framework - external: ecs -- name: threat.technique.id - external: ecs -- name: threat.technique.name - external: ecs -- name: threat.tactic.id - external: ecs -- name: threat.tactic.name - external: ecs diff --git a/packages/panw_cortex_xdr/docs/README.md b/packages/panw_cortex_xdr/docs/README.md index fbdef3471a2..00387009af8 100644 --- a/packages/panw_cortex_xdr/docs/README.md +++ b/packages/panw_cortex_xdr/docs/README.md @@ -133,81 +133,18 @@ An example event for `alerts` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| dns.question.name | The name being queried. If the name field contains non-printable characters (below 32 or above 126), those characters should be represented as escaped base 10 integers (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, and line feeds should be converted to \t, \r, and \n respectively. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | -| email.subject | A brief summary of the topic of the message. | keyword | -| email.subject.text | Multi-field of `email.subject`. | match_only_text | -| email.to.address | The email address of recipient | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.reason | Reason why this event happened, according to the source. This describes the why of a particular action or outcome captured in the event. Where `event.action` captures the action from the event, `event.reason` describes why that action was taken. For example, a web proxy with an `event.action` which denied the request may also populate `event.reason` with the reason why (e.g. `blocked site`). | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.hash.md5 | MD5 hash. | keyword | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| observer.egress.interface.name | Interface name as reported by the system. | keyword | -| observer.ingress.interface.name | Interface name as reported by the system. | keyword | -| observer.serial_number | Observer serial number. | keyword | | panw_cortex.xdr.action_pretty | Pretty description of the action type. | keyword | | panw_cortex.xdr.agent_data_collection_status | Collection status of the agent. | boolean | | panw_cortex.xdr.agent_ip_addresses_v6 | Agent ipv6 address | ip | @@ -293,60 +230,6 @@ An example event for `alerts` looks as following: | panw_cortex.xdr.resolution_status | | keyword | | panw_cortex.xdr.source | | keyword | | panw_cortex.xdr.starred | If alert type is prioritized (starred). | boolean | -| process.code_signature.status | Additional information about the certificate status. This is useful for logging cryptographic errors with the certificate validity or trust status. Leave unpopulated if the validity or trust of the certificate was unchecked. | keyword | -| process.code_signature.subject_name | Subject name of the code signer | keyword | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.executable | Absolute path to the process executable. | keyword | -| process.executable.text | Multi-field of `process.executable`. | match_only_text | -| process.hash.md5 | MD5 hash. | keyword | -| process.hash.sha256 | SHA256 hash. | keyword | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.parent.code_signature.status | Additional information about the certificate status. This is useful for logging cryptographic errors with the certificate validity or trust status. Leave unpopulated if the validity or trust of the certificate was unchecked. | keyword | -| process.parent.code_signature.subject_name | Subject name of the code signer | keyword | -| process.parent.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.parent.command_line.text | Multi-field of `process.parent.command_line`. | match_only_text | -| process.parent.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.parent.executable | Absolute path to the process executable. | keyword | -| process.parent.executable.text | Multi-field of `process.parent.executable`. | match_only_text | -| process.parent.hash.md5 | MD5 hash. | keyword | -| process.parent.hash.sha256 | SHA256 hash. | keyword | -| process.parent.name | Process name. Sometimes called program name or similar. | keyword | -| process.parent.name.text | Multi-field of `process.parent.name`. | match_only_text | -| process.parent.uptime | Seconds the process has been up. | long | -| process.pid | Process id. | long | -| process.thread.id | Thread ID. | long | -| registry.data.strings | Content when writing string types. Populated as an array when writing string data to the registry. For single string registry types (REG_SZ, REG_EXPAND_SZ), this should be an array with one string. For sequences of string with REG_MULTI_SZ, this array will be variable length. For numeric data, such as REG_DWORD and REG_QWORD, this should be populated with the decimal representation (e.g `"1"`). | wildcard | -| registry.key | Hive-relative path of keys. | keyword | -| registry.path | Full path, including hive, key and value | keyword | -| registry.value | Name of the value written. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| threat.framework | Name of the threat framework used to further categorize and classify the tactic and technique of the reported threat. Framework classification can be provided by detecting systems, evaluated at ingest time, or retrospectively tagged to events. | keyword | -| threat.tactic.id | The id of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/ ) | keyword | -| threat.tactic.name | Name of the type of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/) | keyword | -| threat.technique.id | The id of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.name | The name of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.name.text | Multi-field of `threat.technique.name`. | match_only_text | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | ### Incidents @@ -487,47 +370,18 @@ An example event for `incidents` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | event.dataset | Event dataset | constant_keyword | | event.module | Event module | constant_keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | panw_cortex.xdr.aggregated_score | Aggregated incident score. | long | | panw_cortex.xdr.alert_categories | Categories for alerts contained in the incident. | keyword | | panw_cortex.xdr.alert_count | Count of alerts. | long | @@ -563,21 +417,5 @@ An example event for `incidents` looks as following: | panw_cortex.xdr.users | Usernames related to the incident. | keyword | | panw_cortex.xdr.wildfire_hits | Count of Wildfire hits. | long | | panw_cortex.xdr.xdr_url | URL to Cortex XDR incident. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| threat.framework | Name of the threat framework used to further categorize and classify the tactic and technique of the reported threat. Framework classification can be provided by detecting systems, evaluated at ingest time, or retrospectively tagged to events. | keyword | -| threat.tactic.id | The id of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/ ) | keyword | -| threat.tactic.name | Name of the type of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/) | keyword | -| threat.technique.id | The id of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.name | The name of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.name.text | Multi-field of `threat.technique.name`. | match_only_text | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | diff --git a/packages/panw_cortex_xdr/manifest.yml b/packages/panw_cortex_xdr/manifest.yml index e8d73203476..ef3d253b448 100644 --- a/packages/panw_cortex_xdr/manifest.yml +++ b/packages/panw_cortex_xdr/manifest.yml @@ -1,13 +1,13 @@ name: panw_cortex_xdr title: Palo Alto Cortex XDR -version: "1.26.0" +version: "1.27.0" description: Collect logs from Palo Alto Cortex XDR with Elastic Agent. type: integration format_version: "3.0.2" categories: [security, edr_xdr] conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" icons: - src: /img/icon-cortex.svg title: Palo Alto From 4360e896f9e9e5f8b1b230494927e8e7d9507e38 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:25 +0930 Subject: [PATCH 073/121] [ping_one] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/ping_one --- packages/ping_one/changelog.yml | 5 + .../data_stream/audit/fields/agent.yml | 147 ------------------ .../ping_one/data_stream/audit/fields/ecs.yml | 38 ----- packages/ping_one/docs/README.md | 48 ------ packages/ping_one/manifest.yml | 4 +- 5 files changed, 7 insertions(+), 235 deletions(-) delete mode 100644 packages/ping_one/data_stream/audit/fields/ecs.yml diff --git a/packages/ping_one/changelog.yml b/packages/ping_one/changelog.yml index c8c6807d147..8c26ebe7d79 100644 --- a/packages/ping_one/changelog.yml +++ b/packages/ping_one/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.16.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.15.0" changes: - description: Improve handling of empty responses. diff --git a/packages/ping_one/data_stream/audit/fields/agent.yml b/packages/ping_one/data_stream/audit/fields/agent.yml index bb99e5f0b19..d3d659d48f2 100644 --- a/packages/ping_one/data_stream/audit/fields/agent.yml +++ b/packages/ping_one/data_stream/audit/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: >- diff --git a/packages/ping_one/data_stream/audit/fields/ecs.yml b/packages/ping_one/data_stream/audit/fields/ecs.yml deleted file mode 100644 index 3f6d623b9f4..00000000000 --- a/packages/ping_one/data_stream/audit/fields/ecs.yml +++ /dev/null @@ -1,38 +0,0 @@ -- external: ecs - name: client.user.id -- external: ecs - name: client.user.name -- external: ecs - name: ecs.version -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.type -- external: ecs - name: related.user -- external: ecs - name: tags -- external: ecs - name: url.domain -- external: ecs - name: url.original -- external: ecs - name: url.path -- external: ecs - name: url.scheme -- external: ecs - name: user.id -- external: ecs - name: user.name diff --git a/packages/ping_one/docs/README.md b/packages/ping_one/docs/README.md index 72d95c0e7e1..3fac7610e65 100644 --- a/packages/ping_one/docs/README.md +++ b/packages/ping_one/docs/README.md @@ -246,53 +246,15 @@ An example event for `audit` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| client.user.id | Unique identifier of the user. | keyword | -| client.user.name | Short name or login of the user. | keyword | -| client.user.name.text | Multi-field of `client.user.name`. | match_only_text | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | ping_one.audit.action.description | A string that specifies the description of the action performed. | text | @@ -323,14 +285,4 @@ An example event for `audit` looks as following: | ping_one.audit.result.id | A string that specifies the ID for the result of the operation. | keyword | | ping_one.audit.result.status | A string that specifies the result of the operation. Options are succeeded or failed. | keyword | | ping_one.audit.tags | A string identifying the activity as the action of an administrator on other administrators. | keyword | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | diff --git a/packages/ping_one/manifest.yml b/packages/ping_one/manifest.yml index b13ce0ba25f..be84b0c68a0 100644 --- a/packages/ping_one/manifest.yml +++ b/packages/ping_one/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: ping_one title: PingOne -version: "1.15.0" +version: "1.16.0" description: Collect logs from PingOne with Elastic-Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - iam conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" screenshots: - src: /img/ping-one-dashboard.png title: PingOne Audit Dashboard Screenshot From 210129896e1f3f11d7e141f19678160b3b46db16 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:26 +0930 Subject: [PATCH 074/121] [pps] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.0.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/pps --- packages/pps/changelog.yml | 5 +++++ packages/pps/data_stream/log/fields/ecs.yml | 20 -------------------- packages/pps/docs/README.md | 11 ----------- packages/pps/manifest.yml | 4 ++-- 4 files changed, 7 insertions(+), 33 deletions(-) delete mode 100644 packages/pps/data_stream/log/fields/ecs.yml diff --git a/packages/pps/changelog.yml b/packages/pps/changelog.yml index 375181b767a..ac711a3b209 100644 --- a/packages/pps/changelog.yml +++ b/packages/pps/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.1.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "0.0.1" changes: - description: Initial Integration for Pleasant Password Server in Elastic diff --git a/packages/pps/data_stream/log/fields/ecs.yml b/packages/pps/data_stream/log/fields/ecs.yml deleted file mode 100644 index e8b886726b1..00000000000 --- a/packages/pps/data_stream/log/fields/ecs.yml +++ /dev/null @@ -1,20 +0,0 @@ -- external: ecs - name: user.name -- external: ecs - name: user.domain -- external: ecs - name: user.email -- external: ecs - name: client.ip -- external: ecs - name: ecs.version -- external: ecs - name: log.syslog.priority -- external: ecs - name: message -- external: ecs - name: event.outcome -- external: ecs - name: log.file.path -- external: ecs - name: tags diff --git a/packages/pps/docs/README.md b/packages/pps/docs/README.md index 0276d532ab7..6528e7fe72b 100644 --- a/packages/pps/docs/README.md +++ b/packages/pps/docs/README.md @@ -109,21 +109,10 @@ An example event for `log` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | | input.type | Input type | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | Log offset | long | | log.source.address | Log source address | keyword | -| log.syslog.priority | Syslog numeric priority of the event, if available. According to RFCs 5424 and 3164, the priority is 8 \* facility + severity. This number is therefore expected to contain a value between 0 and 191. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.email | User email address. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | diff --git a/packages/pps/manifest.yml b/packages/pps/manifest.yml index 9b9c66d40e4..a92dd9882ef 100644 --- a/packages/pps/manifest.yml +++ b/packages/pps/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.3 name: pps title: "Pleasant Password Server" -version: 0.0.1 +version: "0.1.0" source: license: "Apache-2.0" description: "Integration for Pleasant Password Server Syslog Messages" @@ -12,7 +12,7 @@ categories: - security conditions: kibana: - version: "^8.0.0" + version: "^8.13.0" elastic: subscription: "basic" icons: From 0ef94f0308ae2ba65fe19b687ca0da4a47c76945 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:31 +0930 Subject: [PATCH 075/121] [prisma_cloud] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/prisma_cloud --- packages/prisma_cloud/_dev/build/build.yml | 1 - packages/prisma_cloud/changelog.yml | 5 +++++ packages/prisma_cloud/data_stream/alert/fields/beats.yml | 3 --- packages/prisma_cloud/data_stream/audit/fields/beats.yml | 3 --- packages/prisma_cloud/data_stream/host/fields/beats.yml | 3 --- .../prisma_cloud/data_stream/host_profile/fields/beats.yml | 3 --- .../prisma_cloud/data_stream/incident_audit/fields/beats.yml | 3 --- packages/prisma_cloud/docs/README.md | 5 ----- packages/prisma_cloud/manifest.yml | 4 ++-- 9 files changed, 7 insertions(+), 23 deletions(-) diff --git a/packages/prisma_cloud/_dev/build/build.yml b/packages/prisma_cloud/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/prisma_cloud/_dev/build/build.yml +++ b/packages/prisma_cloud/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/prisma_cloud/changelog.yml b/packages/prisma_cloud/changelog.yml index 20167de590b..6d4b62adcb5 100644 --- a/packages/prisma_cloud/changelog.yml +++ b/packages/prisma_cloud/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.3.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.2.0" changes: - description: Update manifest format version to v3.0.3. diff --git a/packages/prisma_cloud/data_stream/alert/fields/beats.yml b/packages/prisma_cloud/data_stream/alert/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/prisma_cloud/data_stream/alert/fields/beats.yml +++ b/packages/prisma_cloud/data_stream/alert/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/prisma_cloud/data_stream/audit/fields/beats.yml b/packages/prisma_cloud/data_stream/audit/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/prisma_cloud/data_stream/audit/fields/beats.yml +++ b/packages/prisma_cloud/data_stream/audit/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/prisma_cloud/data_stream/host/fields/beats.yml b/packages/prisma_cloud/data_stream/host/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/prisma_cloud/data_stream/host/fields/beats.yml +++ b/packages/prisma_cloud/data_stream/host/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/prisma_cloud/data_stream/host_profile/fields/beats.yml b/packages/prisma_cloud/data_stream/host_profile/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/prisma_cloud/data_stream/host_profile/fields/beats.yml +++ b/packages/prisma_cloud/data_stream/host_profile/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/prisma_cloud/data_stream/incident_audit/fields/beats.yml b/packages/prisma_cloud/data_stream/incident_audit/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/prisma_cloud/data_stream/incident_audit/fields/beats.yml +++ b/packages/prisma_cloud/data_stream/incident_audit/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/prisma_cloud/docs/README.md b/packages/prisma_cloud/docs/README.md index 55ff2c3ac7a..0da49e2d461 100644 --- a/packages/prisma_cloud/docs/README.md +++ b/packages/prisma_cloud/docs/README.md @@ -480,7 +480,6 @@ An example event for `alert` looks as following: | prisma_cloud.alert.status | | keyword | | prisma_cloud.alert.time | Timestamp when alert was last reopened for resource update, or the same as firstSeen if there are no status changes. | date | | prisma_cloud.alert.triggered_by | | keyword | -| tags | User defined tags. | keyword | ### Audit @@ -596,7 +595,6 @@ An example event for `audit` looks as following: | prisma_cloud.audit.result | | keyword | | prisma_cloud.audit.timestamp | Timestamp. | date | | prisma_cloud.audit.user | User. | keyword | -| tags | User defined tags. | keyword | ### Host @@ -1575,7 +1573,6 @@ An example event for `host` looks as following: | prisma_cloud.host.wild_fire_usage.bytes | Bytes is the total number of bytes uploaded to the WildFire API. | long | | prisma_cloud.host.wild_fire_usage.queries | Queries is the number of queries to the WildFire API. | long | | prisma_cloud.host.wild_fire_usage.uploads | Uploads is the number of uploads to the WildFire API. | long | -| tags | User defined tags. | keyword | ### Host Profile @@ -1724,7 +1721,6 @@ An example event for `host_profile` looks as following: | prisma_cloud.host_profile.ssh_events.time | Time is the time in which the process was added. If the process was modified, Time is the modification time. | date | | prisma_cloud.host_profile.ssh_events.user | User represents the username that started the process. | keyword | | prisma_cloud.host_profile.time | Time is the last time when this profile was modified. | date | -| tags | User defined tags. | keyword | ### Incident Audit @@ -2071,4 +2067,3 @@ An example event for `incident_audit` looks as following: | prisma_cloud.incident_audit.type | Possible values: [host,container,function,appEmbedded,fargate]. | keyword | | prisma_cloud.incident_audit.vm_id | Azure unique VM ID on which the incident was found. | keyword | | prisma_cloud.incident_audit.windows | Windows indicates if defender OS type is Windows. | boolean | -| tags | User defined tags. | keyword | diff --git a/packages/prisma_cloud/manifest.yml b/packages/prisma_cloud/manifest.yml index 24862149bf3..3258649c944 100644 --- a/packages/prisma_cloud/manifest.yml +++ b/packages/prisma_cloud/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.3 name: prisma_cloud title: "Palo Alto Prisma Cloud" -version: "1.2.0" +version: "1.3.0" description: "Collect logs from Prisma Cloud with Elastic Agent." type: integration categories: @@ -9,7 +9,7 @@ categories: - cloudsecurity_cdr conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" elastic: subscription: "basic" screenshots: From de98d929d5c43ae5502e91fc8035c740e091a2bd Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:34 +0930 Subject: [PATCH 076/121] [proofpoint_tap] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/proofpoint_tap --- packages/proofpoint_tap/changelog.yml | 5 + .../clicks_blocked/fields/agent.yml | 147 ---------- .../data_stream/clicks_blocked/fields/ecs.yml | 84 ------ .../data_stream/clicks_blocked/manifest.yml | 10 +- .../clicks_permitted/fields/agent.yml | 147 ---------- .../clicks_permitted/fields/ecs.yml | 84 ------ .../data_stream/clicks_permitted/manifest.yml | 9 +- .../message_blocked/fields/agent.yml | 147 ---------- .../message_blocked/fields/ecs.yml | 72 ----- .../data_stream/message_blocked/manifest.yml | 10 +- .../message_delivered/fields/agent.yml | 147 ---------- .../message_delivered/fields/ecs.yml | 72 ----- .../message_delivered/manifest.yml | 10 +- packages/proofpoint_tap/docs/README.md | 276 ------------------ packages/proofpoint_tap/manifest.yml | 4 +- 15 files changed, 15 insertions(+), 1209 deletions(-) delete mode 100644 packages/proofpoint_tap/data_stream/clicks_blocked/fields/ecs.yml delete mode 100644 packages/proofpoint_tap/data_stream/clicks_permitted/fields/ecs.yml delete mode 100644 packages/proofpoint_tap/data_stream/message_blocked/fields/ecs.yml delete mode 100644 packages/proofpoint_tap/data_stream/message_delivered/fields/ecs.yml diff --git a/packages/proofpoint_tap/changelog.yml b/packages/proofpoint_tap/changelog.yml index 5f1113b2452..edcb4a22309 100644 --- a/packages/proofpoint_tap/changelog.yml +++ b/packages/proofpoint_tap/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.22.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.21.0" changes: - description: Improve query interval documentation to avoid request throttling. diff --git a/packages/proofpoint_tap/data_stream/clicks_blocked/fields/agent.yml b/packages/proofpoint_tap/data_stream/clicks_blocked/fields/agent.yml index d2c2658271b..79b11221868 100644 --- a/packages/proofpoint_tap/data_stream/clicks_blocked/fields/agent.yml +++ b/packages/proofpoint_tap/data_stream/clicks_blocked/fields/agent.yml @@ -5,162 +5,15 @@ footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: "The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier." - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime." - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: "A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: "Hostname of the host. It normally contains what the `hostname` command returns on the host machine." - - name: id - level: core - type: keyword - ignore_above: 1024 - description: "Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: "Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: "Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." - name: containerized type: boolean description: > diff --git a/packages/proofpoint_tap/data_stream/clicks_blocked/fields/ecs.yml b/packages/proofpoint_tap/data_stream/clicks_blocked/fields/ecs.yml deleted file mode 100644 index c49113c9429..00000000000 --- a/packages/proofpoint_tap/data_stream/clicks_blocked/fields/ecs.yml +++ /dev/null @@ -1,84 +0,0 @@ -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.ip -- external: ecs - name: ecs.version -- external: ecs - name: email.from.address -- external: ecs - name: email.message_id -- external: ecs - name: email.to.address -- external: ecs - name: error.message -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: related.ip -- external: ecs - name: source.ip -- external: ecs - name: tags -- external: ecs - name: url.domain -- external: ecs - name: url.extension -- external: ecs - name: url.fragment -- external: ecs - name: url.full -- external: ecs - name: url.original -- external: ecs - name: url.password -- external: ecs - name: url.path -- external: ecs - name: url.port -- external: ecs - name: url.query -- external: ecs - name: url.scheme -- external: ecs - name: url.username -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version diff --git a/packages/proofpoint_tap/data_stream/clicks_blocked/manifest.yml b/packages/proofpoint_tap/data_stream/clicks_blocked/manifest.yml index a09dba7e836..d360e435824 100644 --- a/packages/proofpoint_tap/data_stream/clicks_blocked/manifest.yml +++ b/packages/proofpoint_tap/data_stream/clicks_blocked/manifest.yml @@ -10,12 +10,7 @@ streams: type: text title: Interval description: >- - Interval to fetch data from Proofpoint TAP API. The Proofpoint API applies a rolling 24 hour request limit - to users of the API. The clicks blocked, messages delivered and messages blocked endpoints allow a pooled - 1800 requests over a 24 hour period. After that requests will be throttled. See details - [here](https://help.proofpoint.com/Threat_Insight_Dashboard/API_Documentation/SIEM_API#Throttle_Limits). - To avoid throttling, assuming all data streams are being ingested, the interval should be at least 2m30s. - NOTE: Supported units for this parameter are h/m/s. + Interval to fetch data from Proofpoint TAP API. The Proofpoint API applies a rolling 24 hour request limit to users of the API. The clicks blocked, messages delivered and messages blocked endpoints allow a pooled 1800 requests over a 24 hour period. After that requests will be throttled. See details [here](https://help.proofpoint.com/Threat_Insight_Dashboard/API_Documentation/SIEM_API#Throttle_Limits). To avoid throttling, assuming all data streams are being ingested, the interval should be at least 2m30s. NOTE: Supported units for this parameter are h/m/s. multi: false required: true show_user: true @@ -24,8 +19,7 @@ streams: type: text title: Initial Interval description: >- - How far back to pull the tap data from the Proofpoint TAP API (The initial interval should be a maximum of 7 days). - NOTE: Supported units for this parameter are h/m/s. + How far back to pull the tap data from the Proofpoint TAP API (The initial interval should be a maximum of 7 days). NOTE: Supported units for this parameter are h/m/s. default: 24h multi: false required: true diff --git a/packages/proofpoint_tap/data_stream/clicks_permitted/fields/agent.yml b/packages/proofpoint_tap/data_stream/clicks_permitted/fields/agent.yml index d2c2658271b..79b11221868 100644 --- a/packages/proofpoint_tap/data_stream/clicks_permitted/fields/agent.yml +++ b/packages/proofpoint_tap/data_stream/clicks_permitted/fields/agent.yml @@ -5,162 +5,15 @@ footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: "The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier." - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime." - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: "A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: "Hostname of the host. It normally contains what the `hostname` command returns on the host machine." - - name: id - level: core - type: keyword - ignore_above: 1024 - description: "Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: "Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: "Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." - name: containerized type: boolean description: > diff --git a/packages/proofpoint_tap/data_stream/clicks_permitted/fields/ecs.yml b/packages/proofpoint_tap/data_stream/clicks_permitted/fields/ecs.yml deleted file mode 100644 index c49113c9429..00000000000 --- a/packages/proofpoint_tap/data_stream/clicks_permitted/fields/ecs.yml +++ /dev/null @@ -1,84 +0,0 @@ -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.ip -- external: ecs - name: ecs.version -- external: ecs - name: email.from.address -- external: ecs - name: email.message_id -- external: ecs - name: email.to.address -- external: ecs - name: error.message -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: related.ip -- external: ecs - name: source.ip -- external: ecs - name: tags -- external: ecs - name: url.domain -- external: ecs - name: url.extension -- external: ecs - name: url.fragment -- external: ecs - name: url.full -- external: ecs - name: url.original -- external: ecs - name: url.password -- external: ecs - name: url.path -- external: ecs - name: url.port -- external: ecs - name: url.query -- external: ecs - name: url.scheme -- external: ecs - name: url.username -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version diff --git a/packages/proofpoint_tap/data_stream/clicks_permitted/manifest.yml b/packages/proofpoint_tap/data_stream/clicks_permitted/manifest.yml index 4a37cfbf061..45c7d73839c 100644 --- a/packages/proofpoint_tap/data_stream/clicks_permitted/manifest.yml +++ b/packages/proofpoint_tap/data_stream/clicks_permitted/manifest.yml @@ -10,11 +10,7 @@ streams: type: text title: Interval description: >- - Interval to fetch data from Proofpoint TAP API. The Proofpoint API applies a rolling 24 hour request limit - to users of the API. The clicks permitted endpoint allows 1800 requests over a 24 hour period. After that - requests will be throttled. See details [here](https://help.proofpoint.com/Threat_Insight_Dashboard/API_Documentation/SIEM_API#Throttle_Limits). - To avoid throttling, the interval should be at least 1m. - NOTE: Supported units for this parameter are h/m/s. + Interval to fetch data from Proofpoint TAP API. The Proofpoint API applies a rolling 24 hour request limit to users of the API. The clicks permitted endpoint allows 1800 requests over a 24 hour period. After that requests will be throttled. See details [here](https://help.proofpoint.com/Threat_Insight_Dashboard/API_Documentation/SIEM_API#Throttle_Limits). To avoid throttling, the interval should be at least 1m. NOTE: Supported units for this parameter are h/m/s. multi: false required: true show_user: true @@ -23,8 +19,7 @@ streams: type: text title: Initial Interval description: >- - How far back to pull the tap data from the Proofpoint TAP API (The initial interval should be a maximum of 7 days). - NOTE: Supported units for this parameter are h/m/s. + How far back to pull the tap data from the Proofpoint TAP API (The initial interval should be a maximum of 7 days). NOTE: Supported units for this parameter are h/m/s. default: 24h multi: false required: true diff --git a/packages/proofpoint_tap/data_stream/message_blocked/fields/agent.yml b/packages/proofpoint_tap/data_stream/message_blocked/fields/agent.yml index d2c2658271b..79b11221868 100644 --- a/packages/proofpoint_tap/data_stream/message_blocked/fields/agent.yml +++ b/packages/proofpoint_tap/data_stream/message_blocked/fields/agent.yml @@ -5,162 +5,15 @@ footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: "The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier." - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime." - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: "A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: "Hostname of the host. It normally contains what the `hostname` command returns on the host machine." - - name: id - level: core - type: keyword - ignore_above: 1024 - description: "Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: "Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: "Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." - name: containerized type: boolean description: > diff --git a/packages/proofpoint_tap/data_stream/message_blocked/fields/ecs.yml b/packages/proofpoint_tap/data_stream/message_blocked/fields/ecs.yml deleted file mode 100644 index 0bdb1f5ebba..00000000000 --- a/packages/proofpoint_tap/data_stream/message_blocked/fields/ecs.yml +++ /dev/null @@ -1,72 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: email.attachments -- external: ecs - name: email.attachments.file.hash.md5 -- external: ecs - name: email.attachments.file.hash.sha256 -- external: ecs - name: email.attachments.file.mime_type -- external: ecs - name: email.attachments.file.name -- external: ecs - name: email.cc.address -- external: ecs - name: email.content_type -- external: ecs - name: email.delivery_timestamp -- external: ecs - name: email.from.address -- external: ecs - name: email.message_id -- external: ecs - name: email.reply_to.address -- external: ecs - name: email.sender.address -- external: ecs - name: email.subject -- external: ecs - name: email.to.address -- external: ecs - name: email.x_mailer -- external: ecs - name: error.message -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: related.hash -- external: ecs - name: related.ip -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: tags diff --git a/packages/proofpoint_tap/data_stream/message_blocked/manifest.yml b/packages/proofpoint_tap/data_stream/message_blocked/manifest.yml index 2df8d3a058e..9fe316cfb4b 100644 --- a/packages/proofpoint_tap/data_stream/message_blocked/manifest.yml +++ b/packages/proofpoint_tap/data_stream/message_blocked/manifest.yml @@ -10,12 +10,7 @@ streams: type: text title: Interval description: >- - Interval to fetch data from Proofpoint TAP API. The Proofpoint API applies a rolling 24 hour request limit - to users of the API. The clicks blocked, messages delivered and messages blocked endpoints allow a pooled - 1800 requests over a 24 hour period. After that requests will be throttled. See details - [here](https://help.proofpoint.com/Threat_Insight_Dashboard/API_Documentation/SIEM_API#Throttle_Limits). - To avoid throttling, assuming all data streams are being ingested, the interval should be at least 2m30s. - NOTE: Supported units for this parameter are h/m/s. + Interval to fetch data from Proofpoint TAP API. The Proofpoint API applies a rolling 24 hour request limit to users of the API. The clicks blocked, messages delivered and messages blocked endpoints allow a pooled 1800 requests over a 24 hour period. After that requests will be throttled. See details [here](https://help.proofpoint.com/Threat_Insight_Dashboard/API_Documentation/SIEM_API#Throttle_Limits). To avoid throttling, assuming all data streams are being ingested, the interval should be at least 2m30s. NOTE: Supported units for this parameter are h/m/s. multi: false required: true show_user: true @@ -24,8 +19,7 @@ streams: type: text title: Initial Interval description: >- - How far back to pull the tap data from the Proofpoint TAP API (The initial interval should be a maximum of 7 days). - NOTE: Supported units for this parameter are h/m/s. + How far back to pull the tap data from the Proofpoint TAP API (The initial interval should be a maximum of 7 days). NOTE: Supported units for this parameter are h/m/s. default: 24h multi: false required: true diff --git a/packages/proofpoint_tap/data_stream/message_delivered/fields/agent.yml b/packages/proofpoint_tap/data_stream/message_delivered/fields/agent.yml index d2c2658271b..79b11221868 100644 --- a/packages/proofpoint_tap/data_stream/message_delivered/fields/agent.yml +++ b/packages/proofpoint_tap/data_stream/message_delivered/fields/agent.yml @@ -5,162 +5,15 @@ footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: "The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier." - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime." - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: "A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: "Hostname of the host. It normally contains what the `hostname` command returns on the host machine." - - name: id - level: core - type: keyword - ignore_above: 1024 - description: "Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: "Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: "Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." - name: containerized type: boolean description: > diff --git a/packages/proofpoint_tap/data_stream/message_delivered/fields/ecs.yml b/packages/proofpoint_tap/data_stream/message_delivered/fields/ecs.yml deleted file mode 100644 index 0bdb1f5ebba..00000000000 --- a/packages/proofpoint_tap/data_stream/message_delivered/fields/ecs.yml +++ /dev/null @@ -1,72 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: email.attachments -- external: ecs - name: email.attachments.file.hash.md5 -- external: ecs - name: email.attachments.file.hash.sha256 -- external: ecs - name: email.attachments.file.mime_type -- external: ecs - name: email.attachments.file.name -- external: ecs - name: email.cc.address -- external: ecs - name: email.content_type -- external: ecs - name: email.delivery_timestamp -- external: ecs - name: email.from.address -- external: ecs - name: email.message_id -- external: ecs - name: email.reply_to.address -- external: ecs - name: email.sender.address -- external: ecs - name: email.subject -- external: ecs - name: email.to.address -- external: ecs - name: email.x_mailer -- external: ecs - name: error.message -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: related.hash -- external: ecs - name: related.ip -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: tags diff --git a/packages/proofpoint_tap/data_stream/message_delivered/manifest.yml b/packages/proofpoint_tap/data_stream/message_delivered/manifest.yml index 8b1b8f774c5..db1a833a496 100644 --- a/packages/proofpoint_tap/data_stream/message_delivered/manifest.yml +++ b/packages/proofpoint_tap/data_stream/message_delivered/manifest.yml @@ -10,12 +10,7 @@ streams: type: text title: Interval description: >- - Interval to fetch data from Proofpoint TAP API. The Proofpoint API applies a rolling 24 hour request limit - to users of the API. The clicks blocked, messages delivered and messages blocked endpoints allow a pooled - 1800 requests over a 24 hour period. After that requests will be throttled. See details - [here](https://help.proofpoint.com/Threat_Insight_Dashboard/API_Documentation/SIEM_API#Throttle_Limits). - To avoid throttling, assuming all data streams are being ingested, the interval should be at least 2m30s. - NOTE: Supported units for this parameter are h/m/s. + Interval to fetch data from Proofpoint TAP API. The Proofpoint API applies a rolling 24 hour request limit to users of the API. The clicks blocked, messages delivered and messages blocked endpoints allow a pooled 1800 requests over a 24 hour period. After that requests will be throttled. See details [here](https://help.proofpoint.com/Threat_Insight_Dashboard/API_Documentation/SIEM_API#Throttle_Limits). To avoid throttling, assuming all data streams are being ingested, the interval should be at least 2m30s. NOTE: Supported units for this parameter are h/m/s. multi: false required: true show_user: true @@ -24,8 +19,7 @@ streams: type: text title: Initial Interval description: >- - How far back to pull the tap data from the Proofpoint TAP API (The initial interval should be a maximum of 7 days). - NOTE: Supported units for this parameter are h/m/s. + How far back to pull the tap data from the Proofpoint TAP API (The initial interval should be a maximum of 7 days). NOTE: Supported units for this parameter are h/m/s. default: 24h multi: false required: true diff --git a/packages/proofpoint_tap/docs/README.md b/packages/proofpoint_tap/docs/README.md index 735b37b0220..be3b32f0ad4 100644 --- a/packages/proofpoint_tap/docs/README.md +++ b/packages/proofpoint_tap/docs/README.md @@ -161,63 +161,15 @@ An example event for `clicks_blocked` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | -| email.message_id | Identifier from the RFC 5322 `Message-ID:` email header that refers to a particular email message. | wildcard | -| email.to.address | The email address of recipient | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | proofpoint_tap.clicks_blocked.campaign_id | An identifier for the campaign of which the threat is a member, if available at the time of the query. Threats can be linked to campaigns even after these events are retrieved. | keyword | @@ -229,32 +181,6 @@ An example event for `clicks_blocked` looks as following: | proofpoint_tap.clicks_blocked.threat.time | Proofpoint identified the URL as a threat at this time. | date | | proofpoint_tap.clicks_blocked.threat.url | A link to the entry on the TAP Dashboard for the particular threat. | keyword | | proofpoint_tap.guid | The ID of the message within PPS. It can be used to identify the message in PPS and is guaranteed to be unique. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | -| url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| url.full.text | Multi-field of `url.full`. | match_only_text | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.password | Password of the request. | keyword | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.username | Username of the request. | keyword | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | ### Clicks Permitted @@ -399,63 +325,15 @@ An example event for `clicks_permitted` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | -| email.message_id | Identifier from the RFC 5322 `Message-ID:` email header that refers to a particular email message. | wildcard | -| email.to.address | The email address of recipient | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | proofpoint_tap.clicks_permitted.campaign_id | An identifier for the campaign of which the threat is a member, if available at the time of the query. Threats can be linked to campaigns even after these events are retrieved. | keyword | @@ -467,32 +345,6 @@ An example event for `clicks_permitted` looks as following: | proofpoint_tap.clicks_permitted.threat.time | Proofpoint identified the URL as a threat at this time. | date | | proofpoint_tap.clicks_permitted.threat.url | A link to the entry on the TAP Dashboard for the particular threat. | keyword | | proofpoint_tap.guid | The ID of the message within PPS. It can be used to identify the message in PPS and is guaranteed to be unique. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | -| url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| url.full.text | Multi-field of `url.full`. | match_only_text | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.password | Password of the request. | keyword | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.username | Username of the request. | keyword | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | ### Message Blocked @@ -703,65 +555,15 @@ An example event for `message_blocked` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.attachments | A list of objects describing the attachment files sent along with an email message. | nested | -| email.attachments.file.hash.md5 | MD5 hash. | keyword | -| email.attachments.file.hash.sha256 | SHA256 hash. | keyword | -| email.attachments.file.mime_type | The MIME media type of the attachment. This value will typically be extracted from the `Content-Type` MIME header field. | keyword | -| email.attachments.file.name | Name of the attachment file including the file extension. | keyword | -| email.cc.address | The email address of CC recipient | keyword | -| email.content_type | Information about how the message is to be displayed. Typically a MIME type. | keyword | -| email.delivery_timestamp | The date and time when the email message was received by the service or client. | date | -| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | -| email.message_id | Identifier from the RFC 5322 `Message-ID:` email header that refers to a particular email message. | wildcard | -| email.reply_to.address | The address that replies should be delivered to based on the value in the RFC 5322 `Reply-To:` header. | keyword | -| email.sender.address | Per RFC 5322, specifies the address responsible for the actual transmission of the message. | keyword | -| email.subject | A brief summary of the topic of the message. | keyword | -| email.subject.text | Multi-field of `email.subject`. | match_only_text | -| email.to.address | The email address of recipient | keyword | -| email.x_mailer | The name of the application that was used to draft and send the original email message. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | proofpoint_tap.guid | The ID of the message within PPS. It can be used to identify the message in PPS and is guaranteed to be unique. | keyword | @@ -794,20 +596,6 @@ An example event for `message_blocked` looks as following: | proofpoint_tap.message_blocked.threat_info_map.threat.type | Whether the threat was an attachment, URL, or message type. | keyword | | proofpoint_tap.message_blocked.threat_info_map.threat.url | A link to the entry about the threat on the TAP Dashboard. | keyword | | proofpoint_tap.message_blocked.to_addresses | A list of email addresses contained within the To: header, excluding friendly names. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | ### Message Delivered @@ -944,65 +732,15 @@ An example event for `message_delivered` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.attachments | A list of objects describing the attachment files sent along with an email message. | nested | -| email.attachments.file.hash.md5 | MD5 hash. | keyword | -| email.attachments.file.hash.sha256 | SHA256 hash. | keyword | -| email.attachments.file.mime_type | The MIME media type of the attachment. This value will typically be extracted from the `Content-Type` MIME header field. | keyword | -| email.attachments.file.name | Name of the attachment file including the file extension. | keyword | -| email.cc.address | The email address of CC recipient | keyword | -| email.content_type | Information about how the message is to be displayed. Typically a MIME type. | keyword | -| email.delivery_timestamp | The date and time when the email message was received by the service or client. | date | -| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword | -| email.message_id | Identifier from the RFC 5322 `Message-ID:` email header that refers to a particular email message. | wildcard | -| email.reply_to.address | The address that replies should be delivered to based on the value in the RFC 5322 `Reply-To:` header. | keyword | -| email.sender.address | Per RFC 5322, specifies the address responsible for the actual transmission of the message. | keyword | -| email.subject | A brief summary of the topic of the message. | keyword | -| email.subject.text | Multi-field of `email.subject`. | match_only_text | -| email.to.address | The email address of recipient | keyword | -| email.x_mailer | The name of the application that was used to draft and send the original email message. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | proofpoint_tap.guid | The ID of the message within PPS. It can be used to identify the message in PPS and is guaranteed to be unique. | keyword | @@ -1033,18 +771,4 @@ An example event for `message_delivered` looks as following: | proofpoint_tap.message_delivered.threat_info_map.threat.type | Whether the threat was an attachment, URL, or message type. | keyword | | proofpoint_tap.message_delivered.threat_info_map.threat.url | A link to the entry about the threat on the TAP Dashboard. | keyword | | proofpoint_tap.message_delivered.to_addresses | A list of email addresses contained within the To: header, excluding friendly names. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | diff --git a/packages/proofpoint_tap/manifest.yml b/packages/proofpoint_tap/manifest.yml index 34fbd30c7f3..47aa28ca034 100644 --- a/packages/proofpoint_tap/manifest.yml +++ b/packages/proofpoint_tap/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: proofpoint_tap title: Proofpoint TAP -version: "1.21.0" +version: "1.22.0" description: Collect logs from Proofpoint TAP with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - email_security conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" screenshots: - src: /img/proofpoint_tap-screenshot.png title: Proofpoint TAP blocked clicks dashboard screenshot From 7367e2c60f8ddb4b17bcce85b099ceaabe069db8 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:35 +0930 Subject: [PATCH 077/121] [pulse_connect_secure] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^7.16.0 || ^8.0.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/pulse_connect_secure --- packages/pulse_connect_secure/changelog.yml | 5 + .../data_stream/log/fields/agent.yml | 147 ------------------ .../data_stream/log/fields/ecs.yml | 90 ----------- packages/pulse_connect_secure/docs/README.md | 77 --------- packages/pulse_connect_secure/manifest.yml | 4 +- 5 files changed, 7 insertions(+), 316 deletions(-) diff --git a/packages/pulse_connect_secure/changelog.yml b/packages/pulse_connect_secure/changelog.yml index 4f4f8b74c68..2b88be558c4 100644 --- a/packages/pulse_connect_secure/changelog.yml +++ b/packages/pulse_connect_secure/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.1.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.0.1" changes: - description: Fix sample event. diff --git a/packages/pulse_connect_secure/data_stream/log/fields/agent.yml b/packages/pulse_connect_secure/data_stream/log/fields/agent.yml index 79a7a39864b..31300ef7751 100644 --- a/packages/pulse_connect_secure/data_stream/log/fields/agent.yml +++ b/packages/pulse_connect_secure/data_stream/log/fields/agent.yml @@ -5,162 +5,15 @@ footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." - - name: id - level: core - type: keyword - ignore_above: 1024 - description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." - name: containerized type: boolean description: > diff --git a/packages/pulse_connect_secure/data_stream/log/fields/ecs.yml b/packages/pulse_connect_secure/data_stream/log/fields/ecs.yml index 6a4f52e7d57..adb0dc85322 100644 --- a/packages/pulse_connect_secure/data_stream/log/fields/ecs.yml +++ b/packages/pulse_connect_secure/data_stream/log/fields/ecs.yml @@ -1,92 +1,2 @@ - external: ecs name: '@timestamp' -- external: ecs - name: ecs.version -- external: ecs - name: event.created -- external: ecs - name: event.outcome -- external: ecs - name: client.address -- external: ecs - name: client.as.number -- external: ecs - name: client.as.organization.name -- external: ecs - name: client.geo.region_iso_code -- external: ecs - name: client.geo.region_name -- external: ecs - name: client.geo.city_name -- external: ecs - name: client.geo.continent_name -- external: ecs - name: client.geo.country_iso_code -- external: ecs - name: client.geo.country_name -- external: ecs - name: client.geo.location -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.country_name -- external: ecs - name: client.ip -- external: ecs - name: client.nat.ip -- external: ecs - name: log.syslog.priority -- external: ecs - name: message -- external: ecs - name: network.type -- external: ecs - name: observer.hostname -- external: ecs - name: observer.ip -- external: ecs - name: observer.name -- external: ecs - name: observer.product -- external: ecs - name: observer.type -- external: ecs - name: observer.vendor -- external: ecs - name: tags -- external: ecs - name: user.name -- external: ecs - name: user.domain -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: source.address -- external: ecs - name: source.geo.location -- external: ecs - name: source.ip -- external: ecs - name: source.nat.ip diff --git a/packages/pulse_connect_secure/docs/README.md b/packages/pulse_connect_secure/docs/README.md index b8d0e95d560..215016527e8 100644 --- a/packages/pulse_connect_secure/docs/README.md +++ b/packages/pulse_connect_secure/docs/README.md @@ -142,98 +142,21 @@ An example event for `log` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. | date | -| client.address | Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| client.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| client.as.organization.name | Organization name. | keyword | -| client.as.organization.name.text | Multi-field of `client.as.organization.name`. | match_only_text | -| client.geo.city_name | City name. | keyword | -| client.geo.continent_name | Name of the continent. | keyword | -| client.geo.country_iso_code | Country ISO code. | keyword | -| client.geo.country_name | Country name. | keyword | -| client.geo.location | Longitude and latitude. | geo_point | -| client.geo.region_iso_code | Region ISO code. | keyword | -| client.geo.region_name | Region name. | keyword | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| client.nat.ip | Translated IP of source based NAT sessions (e.g. internal client to internet). Typically connections traversing load balancers, firewalls, or routers. | ip | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.module | Event module | constant_keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| log.syslog.priority | Syslog numeric priority of the event, if available. According to RFCs 5424 and 3164, the priority is 8 \* facility + severity. This number is therefore expected to contain a value between 0 and 191. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | -| observer.hostname | Hostname of the observer. | keyword | -| observer.ip | IP addresses of the observer. | ip | -| observer.name | Custom name of the observer. This is a name that can be given to an observer. This can be helpful for example if multiple firewalls of the same model are used in an organization. If no custom name is needed, the field can be left empty. | keyword | -| observer.product | The product name of the observer. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | | pulse_secure.realm | test | keyword | | pulse_secure.role | test | keyword | | pulse_secure.session.id | test | keyword | | pulse_secure.session.id_short | | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.nat.ip | Translated ip of source based NAT sessions (e.g. internal client to internet) Typically connections traversing load balancers, firewalls, or routers. | ip | -| tags | List of keywords used to tag each event. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | diff --git a/packages/pulse_connect_secure/manifest.yml b/packages/pulse_connect_secure/manifest.yml index b862911c584..199dcd74163 100644 --- a/packages/pulse_connect_secure/manifest.yml +++ b/packages/pulse_connect_secure/manifest.yml @@ -1,6 +1,6 @@ name: pulse_connect_secure title: Pulse Connect Secure -version: 2.0.1 +version: "2.1.0" description: Collect logs from Pulse Connect Secure with Elastic Agent. type: integration icons: @@ -12,7 +12,7 @@ format_version: "3.0.3" categories: [vpn_security, security] conditions: kibana: - version: "^7.16.0 || ^8.0.0" + version: "^8.13.0" policy_templates: - name: pulse_connect_secure title: Pulse Connect Secure logs From 456dddc469fbbd16f6883aea45c5ea4f0d93430c Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:38 +0930 Subject: [PATCH 078/121] [qualys_vmdr] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/qualys_vmdr --- packages/qualys_vmdr/_dev/build/build.yml | 1 - packages/qualys_vmdr/changelog.yml | 5 +++ .../asset_host_detection/fields/beats.yml | 3 -- .../knowledge_base/fields/beats.yml | 3 -- .../data_stream/user_activity/fields/ecs.yml | 32 ------------------- packages/qualys_vmdr/docs/README.md | 20 ------------ packages/qualys_vmdr/manifest.yml | 4 +-- 7 files changed, 7 insertions(+), 61 deletions(-) delete mode 100644 packages/qualys_vmdr/data_stream/user_activity/fields/ecs.yml diff --git a/packages/qualys_vmdr/_dev/build/build.yml b/packages/qualys_vmdr/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/qualys_vmdr/_dev/build/build.yml +++ b/packages/qualys_vmdr/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/qualys_vmdr/changelog.yml b/packages/qualys_vmdr/changelog.yml index 83ce50d1c20..83749736bb3 100644 --- a/packages/qualys_vmdr/changelog.yml +++ b/packages/qualys_vmdr/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.3.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "3.2.2" changes: - description: Fix date format to match user activity API behaviour. diff --git a/packages/qualys_vmdr/data_stream/asset_host_detection/fields/beats.yml b/packages/qualys_vmdr/data_stream/asset_host_detection/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/qualys_vmdr/data_stream/asset_host_detection/fields/beats.yml +++ b/packages/qualys_vmdr/data_stream/asset_host_detection/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/qualys_vmdr/data_stream/knowledge_base/fields/beats.yml b/packages/qualys_vmdr/data_stream/knowledge_base/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/qualys_vmdr/data_stream/knowledge_base/fields/beats.yml +++ b/packages/qualys_vmdr/data_stream/knowledge_base/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/qualys_vmdr/data_stream/user_activity/fields/ecs.yml b/packages/qualys_vmdr/data_stream/user_activity/fields/ecs.yml deleted file mode 100644 index d033a9239be..00000000000 --- a/packages/qualys_vmdr/data_stream/user_activity/fields/ecs.yml +++ /dev/null @@ -1,32 +0,0 @@ -- name: event.action - external: ecs -- name: event.provider - external: ecs -- name: message - external: ecs -- name: related.ip - external: ecs -- name: related.user - external: ecs -- name: source.as.number - external: ecs -- name: source.as.organization.name - external: ecs -- name: source.geo.city_name - external: ecs -- name: source.geo.continent_name - external: ecs -- name: source.geo.country_iso_code - external: ecs -- name: source.geo.country_name - external: ecs -- name: source.geo.region_iso_code - external: ecs -- name: source.geo.region_name - external: ecs -- name: source.ip - external: ecs -- name: user.name - external: ecs -- name: user.roles - external: ecs diff --git a/packages/qualys_vmdr/docs/README.md b/packages/qualys_vmdr/docs/README.md index 1884863a077..31c469d2d95 100644 --- a/packages/qualys_vmdr/docs/README.md +++ b/packages/qualys_vmdr/docs/README.md @@ -315,7 +315,6 @@ An example event for `asset_host_detection` looks as following: | qualys_vmdr.asset_host_detection.vulnerability.times.reopened | | long | | qualys_vmdr.asset_host_detection.vulnerability.type | | keyword | | qualys_vmdr.asset_host_detection.vulnerability.unique_vuln_id | | keyword | -| tags | User defined tags. | keyword | ### Knowledge Base @@ -506,7 +505,6 @@ An example event for `knowledge_base` looks as following: | qualys_vmdr.knowledge_base.vendor_reference_list.id | | keyword | | qualys_vmdr.knowledge_base.vendor_reference_list.url | | keyword | | qualys_vmdr.knowledge_base.vuln_type | | keyword | -| tags | User defined tags. | keyword | ### User Activity @@ -604,12 +602,9 @@ An example event for `user_activity` looks as following: | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | | event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | constant_keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | | input.type | Type of filebeat input. | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | qualys_vmdr.user_activity.Action | | keyword | | qualys_vmdr.user_activity.Date | | date | | qualys_vmdr.user_activity.Details | | keyword | @@ -617,18 +612,3 @@ An example event for `user_activity` looks as following: | qualys_vmdr.user_activity.User_IP | | keyword | | qualys_vmdr.user_activity.User_Name | | keyword | | qualys_vmdr.user_activity.User_Role | | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user.roles | Array of user roles at the time of the event. | keyword | diff --git a/packages/qualys_vmdr/manifest.yml b/packages/qualys_vmdr/manifest.yml index dedbd6beecd..80f6b908706 100644 --- a/packages/qualys_vmdr/manifest.yml +++ b/packages/qualys_vmdr/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: qualys_vmdr title: Qualys VMDR -version: "3.2.2" +version: "3.3.0" description: Collect data from Qualys VMDR platform with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - vulnerability_management conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: subscription: basic screenshots: From 11cf779f41d0f1e277308360787da804ded499a9 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:40 +0930 Subject: [PATCH 079/121] [rapid7_insightvm] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/rapid7_insightvm --- packages/rapid7_insightvm/_dev/build/build.yml | 1 - packages/rapid7_insightvm/changelog.yml | 5 +++++ packages/rapid7_insightvm/data_stream/asset/fields/beats.yml | 3 --- .../data_stream/vulnerability/fields/beats.yml | 3 --- packages/rapid7_insightvm/docs/README.md | 2 -- packages/rapid7_insightvm/manifest.yml | 4 ++-- 6 files changed, 7 insertions(+), 11 deletions(-) diff --git a/packages/rapid7_insightvm/_dev/build/build.yml b/packages/rapid7_insightvm/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/rapid7_insightvm/_dev/build/build.yml +++ b/packages/rapid7_insightvm/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/rapid7_insightvm/changelog.yml b/packages/rapid7_insightvm/changelog.yml index 2359aad9e50..3d0595bf035 100644 --- a/packages/rapid7_insightvm/changelog.yml +++ b/packages/rapid7_insightvm/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.12.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.11.0" changes: - description: Improve handling of empty responses. diff --git a/packages/rapid7_insightvm/data_stream/asset/fields/beats.yml b/packages/rapid7_insightvm/data_stream/asset/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/rapid7_insightvm/data_stream/asset/fields/beats.yml +++ b/packages/rapid7_insightvm/data_stream/asset/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/rapid7_insightvm/data_stream/vulnerability/fields/beats.yml b/packages/rapid7_insightvm/data_stream/vulnerability/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/rapid7_insightvm/data_stream/vulnerability/fields/beats.yml +++ b/packages/rapid7_insightvm/data_stream/vulnerability/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/rapid7_insightvm/docs/README.md b/packages/rapid7_insightvm/docs/README.md index 148c69a30a9..445de560009 100644 --- a/packages/rapid7_insightvm/docs/README.md +++ b/packages/rapid7_insightvm/docs/README.md @@ -226,7 +226,6 @@ An example event for `asset` looks as following: | rapid7.insightvm.asset.type | Enum: "hypervisor" "mobile" "guest" "physical" "unknown" The type of asset. | keyword | | rapid7.insightvm.asset.unique_identifiers.id | The unique identifier. | keyword | | rapid7.insightvm.asset.unique_identifiers.source | The source of the unique identifier. | keyword | -| tags | User defined tags. | keyword | ### vulnerability @@ -481,4 +480,3 @@ An example event for `vulnerability` looks as following: | rapid7.insightvm.vulnerability.severity | Enum: "critical" "low" "severe" "informational" "none" "moderate" The severity of the vulnerability. | keyword | | rapid7.insightvm.vulnerability.severity_score | The severity score of the vulnerability, on a scale of 0-10. | long | | rapid7.insightvm.vulnerability.title | The title (summary) of the vulnerability. | keyword | -| tags | User defined tags. | keyword | diff --git a/packages/rapid7_insightvm/manifest.yml b/packages/rapid7_insightvm/manifest.yml index 1ee759938d8..7a9a58b0160 100644 --- a/packages/rapid7_insightvm/manifest.yml +++ b/packages/rapid7_insightvm/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: rapid7_insightvm title: Rapid7 InsightVM -version: "1.11.0" +version: "1.12.0" source: license: "Elastic-2.0" description: Collect logs from Rapid7 InsightVM with Elastic Agent. @@ -11,7 +11,7 @@ categories: - vulnerability_management conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: subscription: "basic" screenshots: From c7b0a195ae8034524aebb38ff6ec0aa1279d914d Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:41 +0930 Subject: [PATCH 080/121] [santa] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.7.1 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/santa --- packages/santa/changelog.yml | 5 + .../santa/data_stream/log/fields/agent.yml | 167 +----------------- packages/santa/data_stream/log/fields/ecs.yml | 46 ----- packages/santa/docs/README.md | 54 ------ packages/santa/manifest.yml | 4 +- 5 files changed, 8 insertions(+), 268 deletions(-) delete mode 100644 packages/santa/data_stream/log/fields/ecs.yml diff --git a/packages/santa/changelog.yml b/packages/santa/changelog.yml index f70e6126a03..b852bd1b8b8 100644 --- a/packages/santa/changelog.yml +++ b/packages/santa/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.18.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "3.17.0" changes: - description: Update manifest format version to v3.0.3. diff --git a/packages/santa/data_stream/log/fields/agent.yml b/packages/santa/data_stream/log/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/santa/data_stream/log/fields/agent.yml +++ b/packages/santa/data_stream/log/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/santa/data_stream/log/fields/ecs.yml b/packages/santa/data_stream/log/fields/ecs.yml deleted file mode 100644 index ca4a4858ec6..00000000000 --- a/packages/santa/data_stream/log/fields/ecs.yml +++ /dev/null @@ -1,46 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.ingested -- external: ecs - name: agent.id -- external: ecs - name: file.path -- external: ecs - name: file.target_path -- external: ecs - name: file.x509.issuer.common_name -- external: ecs - name: group.id -- external: ecs - name: group.name -- external: ecs - name: log.file.path -- external: ecs - name: log.level -- external: ecs - name: process.args -- external: ecs - name: process.executable -- external: ecs - name: process.hash.sha256 -- external: ecs - name: process.pid -- external: ecs - name: process.entity_id -- external: ecs - name: process.parent.pid -- external: ecs - name: process.name -- external: ecs - name: process.start -- external: ecs - name: related.hash -- external: ecs - name: related.user -- external: ecs - name: tags -- external: ecs - name: user.id -- external: ecs - name: user.name diff --git a/packages/santa/docs/README.md b/packages/santa/docs/README.md index 57b4d5155e8..a9acbc3336c 100644 --- a/packages/santa/docs/README.md +++ b/packages/santa/docs/README.md @@ -127,67 +127,17 @@ An example event for `log` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| agent.id | Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | | event.module | Event module | constant_keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| file.target_path | Target path for symlinks. | keyword | -| file.target_path.text | Multi-field of `file.target_path`. | match_only_text | -| file.x509.issuer.common_name | List of common name (CN) of issuing certificate authority. | keyword | -| group.id | Unique identifier for the group on the system/platform. | keyword | -| group.name | Name of the group. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | -| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | | log.offset | Log offset | long | -| process.args | Array of process arguments, starting with the absolute path to the executable. May be filtered to protect sensitive information. | keyword | -| process.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.executable | Absolute path to the process executable. | keyword | -| process.executable.text | Multi-field of `process.executable`. | match_only_text | -| process.hash.sha256 | SHA256 hash. | keyword | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.parent.pid | Process id. | long | -| process.pid | Process id. | long | -| process.start | The time the process started. | date | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.user | All the user names or other user identifiers seen on the event. | keyword | | santa.action | Action | keyword | | santa.certificate.common_name | Common name from code signing certificate. | keyword | | santa.certificate.sha256 | SHA256 hash of code signing certificate. | keyword | @@ -204,8 +154,4 @@ An example event for `log` looks as following: | santa.mode | Operating mode of Santa. | keyword | | santa.pidversion | macOS process identity version. | long | | santa.reason | Reason for the decision. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | diff --git a/packages/santa/manifest.yml b/packages/santa/manifest.yml index 242cda09914..077c576812b 100644 --- a/packages/santa/manifest.yml +++ b/packages/santa/manifest.yml @@ -1,6 +1,6 @@ name: santa title: Google Santa -version: "3.17.0" +version: "3.18.0" description: Collect logs from Google Santa with Elastic Agent. type: integration icons: @@ -12,7 +12,7 @@ categories: - security conditions: kibana: - version: ^8.7.1 + version: "^8.13.0" screenshots: - src: /img/kibana-santa-log-overview.png title: kibana santa log overview From ea5802a428565cd822b33d881d63abeb6e53aa02 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:45 +0930 Subject: [PATCH 081/121] [sentinel_one] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/sentinel_one --- .../_dev/deploy/docker/files/config.yml | 4 +- packages/sentinel_one/changelog.yml | 5 + .../data_stream/activity/fields/agent.yml | 147 --------- .../data_stream/activity/fields/ecs.yml | 60 ---- .../data_stream/agent/fields/agent.yml | 147 --------- .../data_stream/agent/fields/ecs.yml | 44 --- .../data_stream/alert/fields/agent.yml | 147 --------- .../data_stream/alert/fields/ecs.yml | 116 ------- .../data_stream/group/fields/agent.yml | 147 --------- .../data_stream/group/fields/ecs.yml | 24 -- .../data_stream/threat/fields/agent.yml | 147 --------- .../data_stream/threat/fields/ecs.yml | 72 ----- packages/sentinel_one/docs/README.md | 305 ------------------ packages/sentinel_one/manifest.yml | 4 +- 14 files changed, 9 insertions(+), 1360 deletions(-) delete mode 100644 packages/sentinel_one/data_stream/activity/fields/ecs.yml delete mode 100644 packages/sentinel_one/data_stream/agent/fields/ecs.yml delete mode 100644 packages/sentinel_one/data_stream/alert/fields/ecs.yml delete mode 100644 packages/sentinel_one/data_stream/group/fields/ecs.yml delete mode 100644 packages/sentinel_one/data_stream/threat/fields/ecs.yml diff --git a/packages/sentinel_one/_dev/deploy/docker/files/config.yml b/packages/sentinel_one/_dev/deploy/docker/files/config.yml index 9412d2423e7..f9abf588bbb 100644 --- a/packages/sentinel_one/_dev/deploy/docker/files/config.yml +++ b/packages/sentinel_one/_dev/deploy/docker/files/config.yml @@ -27,5 +27,5 @@ rules: methods: ["GET"] responses: - status_code: 200 - body: | - {"data":[{"agentDetectionInfo":{"accountId":"1234567890123456789","accountName":"Default","agentDetectionState":null,"agentDomain":"WORKGROUP","agentIpV4":"10.0.0.1","agentIpV6":"2a02:cf40::","agentLastLoggedInUpn":null,"agentLastLoggedInUserMail":null,"agentLastLoggedInUserName":"","agentMitigationMode":"protect","agentOsName":"linux","agentOsRevision":"1234","agentRegisteredAt":"2022-04-06T08:26:45.515278Z","agentUuid":"fwfbxxxxxxxxxxqcfjfnxxxxxxxxx","agentVersion":"21.x.x","cloudProviders":{},"externalIp":"81.2.69.143","groupId":"1234567890123456789","groupName":"Default Group","siteId":"1234567890123456789","siteName":"Default site"},"agentRealtimeInfo":{"accountId":"1234567890123456789","accountName":"Default","activeThreats":7,"agentComputerName":"test-LINUX","agentDecommissionedAt":null,"agentDomain":"WORKGROUP","agentId":"1234567890123456789","agentInfected":true,"agentIsActive":true,"agentIsDecommissioned":false,"agentMachineType":"server","agentMitigationMode":"detect","agentNetworkStatus":"connected","agentOsName":"linux","agentOsRevision":"1234","agentOsType":"linux","agentUuid":"fwfbxxxxxxxxxxqcfjfnxxxxxxxxx","agentVersion":"21.x.x.1234","groupId":"1234567890123456789","groupName":"Default Group","networkInterfaces":[{"id":"1234567890123456789","inet":["10.0.0.1"],"inet6":["2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6"],"name":"Ethernet","physical":"DE:AD:00:00:BE:EF"}],"operationalState":"na","rebootRequired":false,"scanAbortedAt":null,"scanFinishedAt":"2022-04-06T09:18:21.090855Z","scanStartedAt":"2022-04-06T08:26:52.838047Z","scanStatus":"finished","siteId":"1234567890123456789","siteName":"Default site","storageName":null,"storageType":null,"userActionsNeeded":[]},"containerInfo":{"id":null,"image":null,"labels":null,"name":null},"id":"1234567890123456789","indicators":[],"kubernetesInfo":{"cluster":null,"controllerKind":null,"controllerLabels":null,"controllerName":null,"namespace":null,"namespaceLabels":null,"node":null,"pod":null,"podLabels":null},"mitigationStatus":[{"action":"unquarantine","actionsCounters":{"failed":0,"notFound":0,"pendingReboot":0,"success":1,"total":1},"agentSupportsReport":true,"groupNotFound":false,"lastUpdate":"2022-04-06T08:54:17.198002Z","latestReport":"/threats/mitigation-report","mitigationEndedAt":"2022-04-06T08:54:17.101000Z","mitigationStartedAt":"2022-04-06T08:54:17.101000Z","status":"success"},{"action":"kill","actionsCounters":null,"agentSupportsReport":true,"groupNotFound":false,"lastUpdate":"2022-04-06T08:45:55.303355Z","latestReport":null,"mitigationEndedAt":"2022-04-06T08:45:55.297364Z","mitigationStartedAt":"2022-04-06T08:45:55.297363Z","status":"success"}],"threatInfo":{"analystVerdict":"undefined","analystVerdictDescription":"Undefined","automaticallyResolved":false,"browserType":null,"certificateId":"","classification":"Trojan","classificationSource":"Cloud","cloudFilesHashVerdict":"black","collectionId":"1234567890123456789","confidenceLevel":"malicious","createdAt":"2022-04-06T08:45:54.519988Z","detectionEngines":[{"key":"sentinelone_cloud","title":"SentinelOne Cloud"}],"detectionType":"static","engines":["SentinelOne Cloud"],"externalTicketExists":false,"externalTicketId":null,"failedActions":false,"fileExtension":"EXE","fileExtensionType":"Executable","filePath":"default.exe","fileSize":1234,"fileVerificationType":"NotSigned","identifiedAt":"2022-04-06T08:45:53.968000Z","incidentStatus":"unresolved","incidentStatusDescription":"Unresolved","initiatedBy":"agent_policy","initiatedByDescription":"Agent Policy","initiatingUserId":null,"initiatingUsername":null,"isFileless":false,"isValidCertificate":false,"maliciousProcessArguments":null,"md5":null,"mitigatedPreemptively":false,"mitigationStatus":"not_mitigated","mitigationStatusDescription":"Not mitigated","originatorProcess":"default.exe","pendingActions":false,"processUser":"test user","publisherName":"","reachedEventsLimit":false,"rebootRequired":false,"sha1":"aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d","sha256":null,"storyline":"D0XXXXXXXXXXAF4D","threatId":"1234567890123456789","threatName":"default.exe","updatedAt":"2022-04-06T08:54:17.194122Z"},"whiteningOptions":["hash"]},{"agentDetectionInfo":{"accountId":"1234567890123456789","accountName":"Default","agentDetectionState":null,"agentDomain":"WORKGROUP","agentIpV4":"10.0.0.1","agentIpV6":"2a02:cf40::","agentLastLoggedInUpn":null,"agentLastLoggedInUserMail":null,"agentLastLoggedInUserName":"","agentMitigationMode":"detect","agentOsName":"linux","agentOsRevision":"1234","agentRegisteredAt":"2022-04-06T08:26:45.515278Z","agentUuid":"fwfbxxxxxxxxxxqcfjfnxxxxxxxxx","agentVersion":"21.x.x","cloudProviders":{},"externalIp":"81.2.69.143","groupId":"1234567890123456789","groupName":"Default Group","siteId":"1234567890123456789","siteName":"Default site"},"agentRealtimeInfo":{"accountId":"1234567890123456789","accountName":"Default","activeThreats":7,"agentComputerName":"test-LINUX","agentDecommissionedAt":null,"agentDomain":"WORKGROUP","agentId":"1234567890123456789","agentInfected":true,"agentIsActive":true,"agentIsDecommissioned":false,"agentMachineType":"server","agentMitigationMode":"detect","agentNetworkStatus":"connected","agentOsName":"linux","agentOsRevision":"1234","agentOsType":"linux","agentUuid":"fwfbxxxxxxxxxxqcfjfnxxxxxxxxx","agentVersion":"21.x.x.1234","groupId":"1234567890123456789","groupName":"Default Group","networkInterfaces":[{"id":"1234567890123456789","inet":["10.0.0.1"],"inet6":["2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6"],"name":"Ethernet","physical":"DE:AD:00:00:BE:EF"}],"operationalState":"na","rebootRequired":false,"scanAbortedAt":null,"scanFinishedAt":"2022-04-06T09:18:21.090855Z","scanStartedAt":"2022-04-06T08:26:52.838047Z","scanStatus":"finished","siteId":"1234567890123456789","siteName":"Default site","storageName":null,"storageType":null,"userActionsNeeded":[]},"containerInfo":{"id":null,"image":null,"labels":null,"name":null},"id":"1234567890123456789","indicators":[{"category":"General","description":"Detected by the Static Engine","ids":[43],"tactics":[]},{"category":"Exploitation","description":"Document behaves abnormally","ids":[62],"tactics":[{"name":"Execution","source":"DEFAULT","techniques":[{"link":"https://example.com/","name":"T1234"},{"link":"https://example.com/","name":"T1234"},{"link":"https://example.com/","name":"T1234"}]},{"name":"Initial Access","source":"DEFAULT","techniques":[{"link":"https://example.com/","name":"T1234"}]}]},{"category":"Evasion","description":"Indirect command was executed","ids":[427],"tactics":[{"name":"Defense Evasion","source":"DEFAULT","techniques":[{"link":"https://example.com/","name":"T1234"},{"link":"https://example.com/","name":"T1234"}]}]},{"category":"Evasion","description":"Office program ran macro","ids":[434],"tactics":[{"name":"Execution","source":"DEFAULT","techniques":[{"link":"https://example.com","name":"T1234"}]},{"name":"Initial Access","source":"DEFAULT","techniques":[{"link":"https://example.com","name":"T1234"}]},{"name":"Execution","source":"DEFAULT","techniques":[{"link":"https://example.com","name":"T1234"}]}]},{"category":"Evasion","description":"Process wrote to a hidden file section","ids":[169],"tactics":[{"name":"Defense Evasion","source":"DEFAULT","techniques":[{"link":"https://example.com","name":"T1234"}]}]},{"category":"Evasion","description":"Suspicious registry key was created","ids":[171],"tactics":[{"name":"Defense Evasion","source":"DEFAULT","techniques":[{"link":"https://example.com","name":"T1234"}]}]}],"kubernetesInfo":{"cluster":null,"controllerKind":null,"controllerLabels":null,"controllerName":null,"namespace":null,"namespaceLabels":null,"node":null,"pod":null,"podLabels":null},"mitigationStatus":[],"threatInfo":{"analystVerdict":"undefined","analystVerdictDescription":"Undefined","automaticallyResolved":false,"browserType":null,"certificateId":"","classification":"Malware","classificationSource":"Static","cloudFilesHashVerdict":"black","collectionId":"1234567890123456789","confidenceLevel":"malicious","createdAt":"2022-04-06T08:57:34.744922Z","detectionEngines":[{"key":"pre_execution","title":"On-Write Static AI"},{"key":"data_files","title":"Documents, Scripts"}],"detectionType":"dynamic","engines":["Documents, Scripts","On-Write ABC"],"externalTicketExists":false,"externalTicketId":null,"failedActions":false,"fileExtension":"TXT","fileExtensionType":"Document","filePath":"test/path/user","fileSize":238592,"fileVerificationType":"NotSigned","identifiedAt":"2022-04-06T08:57:34.444000Z","incidentStatus":"unresolved","incidentStatusDescription":"Unresolved","initiatedBy":"agent_policy","initiatedByDescription":"Agent Policy","initiatingUserId":null,"initiatingUsername":null,"isFileless":false,"isValidCertificate":false,"maliciousProcessArguments":"test/path/user","md5":null,"mitigatedPreemptively":false,"mitigationStatus":"not_mitigated","mitigationStatusDescription":"Not mitigated","originatorProcess":"default.EXE","pendingActions":false,"processUser":"test_user","publisherName":"","reachedEventsLimit":false,"rebootRequired":false,"sha1":"aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d","sha256":null,"storyline":"7XXXXXXXXXDD5A41","threatId":"123456789","threatName":"Threats","updatedAt":"2022-04-06T08:57:37.672873Z"},"whiteningOptions":["hash","path","file_type"]}],"pagination":{"nextCursor":null,"totalItems":2}} \ No newline at end of file + body: |- + {"data":[{"agentDetectionInfo":{"accountId":"1234567890123456789","accountName":"Default","agentDetectionState":null,"agentDomain":"WORKGROUP","agentIpV4":"10.0.0.1","agentIpV6":"2a02:cf40::","agentLastLoggedInUpn":null,"agentLastLoggedInUserMail":null,"agentLastLoggedInUserName":"","agentMitigationMode":"protect","agentOsName":"linux","agentOsRevision":"1234","agentRegisteredAt":"2022-04-06T08:26:45.515278Z","agentUuid":"fwfbxxxxxxxxxxqcfjfnxxxxxxxxx","agentVersion":"21.x.x","cloudProviders":{},"externalIp":"81.2.69.143","groupId":"1234567890123456789","groupName":"Default Group","siteId":"1234567890123456789","siteName":"Default site"},"agentRealtimeInfo":{"accountId":"1234567890123456789","accountName":"Default","activeThreats":7,"agentComputerName":"test-LINUX","agentDecommissionedAt":null,"agentDomain":"WORKGROUP","agentId":"1234567890123456789","agentInfected":true,"agentIsActive":true,"agentIsDecommissioned":false,"agentMachineType":"server","agentMitigationMode":"detect","agentNetworkStatus":"connected","agentOsName":"linux","agentOsRevision":"1234","agentOsType":"linux","agentUuid":"fwfbxxxxxxxxxxqcfjfnxxxxxxxxx","agentVersion":"21.x.x.1234","groupId":"1234567890123456789","groupName":"Default Group","networkInterfaces":[{"id":"1234567890123456789","inet":["10.0.0.1"],"inet6":["2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6"],"name":"Ethernet","physical":"DE:AD:00:00:BE:EF"}],"operationalState":"na","rebootRequired":false,"scanAbortedAt":null,"scanFinishedAt":"2022-04-06T09:18:21.090855Z","scanStartedAt":"2022-04-06T08:26:52.838047Z","scanStatus":"finished","siteId":"1234567890123456789","siteName":"Default site","storageName":null,"storageType":null,"userActionsNeeded":[]},"containerInfo":{"id":null,"image":null,"labels":null,"name":null},"id":"1234567890123456789","indicators":[],"kubernetesInfo":{"cluster":null,"controllerKind":null,"controllerLabels":null,"controllerName":null,"namespace":null,"namespaceLabels":null,"node":null,"pod":null,"podLabels":null},"mitigationStatus":[{"action":"unquarantine","actionsCounters":{"failed":0,"notFound":0,"pendingReboot":0,"success":1,"total":1},"agentSupportsReport":true,"groupNotFound":false,"lastUpdate":"2022-04-06T08:54:17.198002Z","latestReport":"/threats/mitigation-report","mitigationEndedAt":"2022-04-06T08:54:17.101000Z","mitigationStartedAt":"2022-04-06T08:54:17.101000Z","status":"success"},{"action":"kill","actionsCounters":null,"agentSupportsReport":true,"groupNotFound":false,"lastUpdate":"2022-04-06T08:45:55.303355Z","latestReport":null,"mitigationEndedAt":"2022-04-06T08:45:55.297364Z","mitigationStartedAt":"2022-04-06T08:45:55.297363Z","status":"success"}],"threatInfo":{"analystVerdict":"undefined","analystVerdictDescription":"Undefined","automaticallyResolved":false,"browserType":null,"certificateId":"","classification":"Trojan","classificationSource":"Cloud","cloudFilesHashVerdict":"black","collectionId":"1234567890123456789","confidenceLevel":"malicious","createdAt":"2022-04-06T08:45:54.519988Z","detectionEngines":[{"key":"sentinelone_cloud","title":"SentinelOne Cloud"}],"detectionType":"static","engines":["SentinelOne Cloud"],"externalTicketExists":false,"externalTicketId":null,"failedActions":false,"fileExtension":"EXE","fileExtensionType":"Executable","filePath":"default.exe","fileSize":1234,"fileVerificationType":"NotSigned","identifiedAt":"2022-04-06T08:45:53.968000Z","incidentStatus":"unresolved","incidentStatusDescription":"Unresolved","initiatedBy":"agent_policy","initiatedByDescription":"Agent Policy","initiatingUserId":null,"initiatingUsername":null,"isFileless":false,"isValidCertificate":false,"maliciousProcessArguments":null,"md5":null,"mitigatedPreemptively":false,"mitigationStatus":"not_mitigated","mitigationStatusDescription":"Not mitigated","originatorProcess":"default.exe","pendingActions":false,"processUser":"test user","publisherName":"","reachedEventsLimit":false,"rebootRequired":false,"sha1":"aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d","sha256":null,"storyline":"D0XXXXXXXXXXAF4D","threatId":"1234567890123456789","threatName":"default.exe","updatedAt":"2022-04-06T08:54:17.194122Z"},"whiteningOptions":["hash"]},{"agentDetectionInfo":{"accountId":"1234567890123456789","accountName":"Default","agentDetectionState":null,"agentDomain":"WORKGROUP","agentIpV4":"10.0.0.1","agentIpV6":"2a02:cf40::","agentLastLoggedInUpn":null,"agentLastLoggedInUserMail":null,"agentLastLoggedInUserName":"","agentMitigationMode":"detect","agentOsName":"linux","agentOsRevision":"1234","agentRegisteredAt":"2022-04-06T08:26:45.515278Z","agentUuid":"fwfbxxxxxxxxxxqcfjfnxxxxxxxxx","agentVersion":"21.x.x","cloudProviders":{},"externalIp":"81.2.69.143","groupId":"1234567890123456789","groupName":"Default Group","siteId":"1234567890123456789","siteName":"Default site"},"agentRealtimeInfo":{"accountId":"1234567890123456789","accountName":"Default","activeThreats":7,"agentComputerName":"test-LINUX","agentDecommissionedAt":null,"agentDomain":"WORKGROUP","agentId":"1234567890123456789","agentInfected":true,"agentIsActive":true,"agentIsDecommissioned":false,"agentMachineType":"server","agentMitigationMode":"detect","agentNetworkStatus":"connected","agentOsName":"linux","agentOsRevision":"1234","agentOsType":"linux","agentUuid":"fwfbxxxxxxxxxxqcfjfnxxxxxxxxx","agentVersion":"21.x.x.1234","groupId":"1234567890123456789","groupName":"Default Group","networkInterfaces":[{"id":"1234567890123456789","inet":["10.0.0.1"],"inet6":["2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6"],"name":"Ethernet","physical":"DE:AD:00:00:BE:EF"}],"operationalState":"na","rebootRequired":false,"scanAbortedAt":null,"scanFinishedAt":"2022-04-06T09:18:21.090855Z","scanStartedAt":"2022-04-06T08:26:52.838047Z","scanStatus":"finished","siteId":"1234567890123456789","siteName":"Default site","storageName":null,"storageType":null,"userActionsNeeded":[]},"containerInfo":{"id":null,"image":null,"labels":null,"name":null},"id":"1234567890123456789","indicators":[{"category":"General","description":"Detected by the Static Engine","ids":[43],"tactics":[]},{"category":"Exploitation","description":"Document behaves abnormally","ids":[62],"tactics":[{"name":"Execution","source":"DEFAULT","techniques":[{"link":"https://example.com/","name":"T1234"},{"link":"https://example.com/","name":"T1234"},{"link":"https://example.com/","name":"T1234"}]},{"name":"Initial Access","source":"DEFAULT","techniques":[{"link":"https://example.com/","name":"T1234"}]}]},{"category":"Evasion","description":"Indirect command was executed","ids":[427],"tactics":[{"name":"Defense Evasion","source":"DEFAULT","techniques":[{"link":"https://example.com/","name":"T1234"},{"link":"https://example.com/","name":"T1234"}]}]},{"category":"Evasion","description":"Office program ran macro","ids":[434],"tactics":[{"name":"Execution","source":"DEFAULT","techniques":[{"link":"https://example.com","name":"T1234"}]},{"name":"Initial Access","source":"DEFAULT","techniques":[{"link":"https://example.com","name":"T1234"}]},{"name":"Execution","source":"DEFAULT","techniques":[{"link":"https://example.com","name":"T1234"}]}]},{"category":"Evasion","description":"Process wrote to a hidden file section","ids":[169],"tactics":[{"name":"Defense Evasion","source":"DEFAULT","techniques":[{"link":"https://example.com","name":"T1234"}]}]},{"category":"Evasion","description":"Suspicious registry key was created","ids":[171],"tactics":[{"name":"Defense Evasion","source":"DEFAULT","techniques":[{"link":"https://example.com","name":"T1234"}]}]}],"kubernetesInfo":{"cluster":null,"controllerKind":null,"controllerLabels":null,"controllerName":null,"namespace":null,"namespaceLabels":null,"node":null,"pod":null,"podLabels":null},"mitigationStatus":[],"threatInfo":{"analystVerdict":"undefined","analystVerdictDescription":"Undefined","automaticallyResolved":false,"browserType":null,"certificateId":"","classification":"Malware","classificationSource":"Static","cloudFilesHashVerdict":"black","collectionId":"1234567890123456789","confidenceLevel":"malicious","createdAt":"2022-04-06T08:57:34.744922Z","detectionEngines":[{"key":"pre_execution","title":"On-Write Static AI"},{"key":"data_files","title":"Documents, Scripts"}],"detectionType":"dynamic","engines":["Documents, Scripts","On-Write ABC"],"externalTicketExists":false,"externalTicketId":null,"failedActions":false,"fileExtension":"TXT","fileExtensionType":"Document","filePath":"test/path/user","fileSize":238592,"fileVerificationType":"NotSigned","identifiedAt":"2022-04-06T08:57:34.444000Z","incidentStatus":"unresolved","incidentStatusDescription":"Unresolved","initiatedBy":"agent_policy","initiatedByDescription":"Agent Policy","initiatingUserId":null,"initiatingUsername":null,"isFileless":false,"isValidCertificate":false,"maliciousProcessArguments":"test/path/user","md5":null,"mitigatedPreemptively":false,"mitigationStatus":"not_mitigated","mitigationStatusDescription":"Not mitigated","originatorProcess":"default.EXE","pendingActions":false,"processUser":"test_user","publisherName":"","reachedEventsLimit":false,"rebootRequired":false,"sha1":"aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d","sha256":null,"storyline":"7XXXXXXXXXDD5A41","threatId":"123456789","threatName":"Threats","updatedAt":"2022-04-06T08:57:37.672873Z"},"whiteningOptions":["hash","path","file_type"]}],"pagination":{"nextCursor":null,"totalItems":2}} diff --git a/packages/sentinel_one/changelog.yml b/packages/sentinel_one/changelog.yml index 86a49d43fef..cfd4f04a431 100644 --- a/packages/sentinel_one/changelog.yml +++ b/packages/sentinel_one/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.24.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.23.3" changes: - description: Fix sample event MAC address. diff --git a/packages/sentinel_one/data_stream/activity/fields/agent.yml b/packages/sentinel_one/data_stream/activity/fields/agent.yml index 6e1bac042bc..894e6f12be2 100644 --- a/packages/sentinel_one/data_stream/activity/fields/agent.yml +++ b/packages/sentinel_one/data_stream/activity/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/sentinel_one/data_stream/activity/fields/ecs.yml b/packages/sentinel_one/data_stream/activity/fields/ecs.yml deleted file mode 100644 index bf0722a2f69..00000000000 --- a/packages/sentinel_one/data_stream/activity/fields/ecs.yml +++ /dev/null @@ -1,60 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: file.hash.sha1 -- external: ecs - name: file.name -- external: ecs - name: file.path -- external: ecs - name: host.geo.city_name -- external: ecs - name: host.geo.continent_name -- external: ecs - name: host.geo.country_iso_code -- external: ecs - name: host.geo.country_name -- external: ecs - name: host.geo.location -- external: ecs - name: host.geo.region_iso_code -- external: ecs - name: host.geo.region_name -- external: ecs - name: message -- external: ecs - name: observer.version -- external: ecs - name: observer.os.family -- external: ecs - name: process.hash.sha1 -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: tags -- external: ecs - name: user.email -- external: ecs - name: user.full_name -- external: ecs - name: user.group.id -- external: ecs - name: user.group.name -- external: ecs - name: user.id diff --git a/packages/sentinel_one/data_stream/agent/fields/agent.yml b/packages/sentinel_one/data_stream/agent/fields/agent.yml index 6e1bac042bc..894e6f12be2 100644 --- a/packages/sentinel_one/data_stream/agent/fields/agent.yml +++ b/packages/sentinel_one/data_stream/agent/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/sentinel_one/data_stream/agent/fields/ecs.yml b/packages/sentinel_one/data_stream/agent/fields/ecs.yml deleted file mode 100644 index eea36baf80e..00000000000 --- a/packages/sentinel_one/data_stream/agent/fields/ecs.yml +++ /dev/null @@ -1,44 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: group.id -- external: ecs - name: group.name -- external: ecs - name: host.geo.city_name -- external: ecs - name: host.geo.continent_name -- external: ecs - name: host.geo.country_iso_code -- external: ecs - name: host.geo.country_name -- external: ecs - name: host.geo.location -- external: ecs - name: host.geo.region_iso_code -- external: ecs - name: host.geo.region_name -- external: ecs - name: host.os.type -- external: ecs - name: observer.version -- external: ecs - name: related.ip -- external: ecs - name: related.hosts -- external: ecs - name: related.user -- external: ecs - name: tags -- external: ecs - name: user.name diff --git a/packages/sentinel_one/data_stream/alert/fields/agent.yml b/packages/sentinel_one/data_stream/alert/fields/agent.yml index 6e1bac042bc..894e6f12be2 100644 --- a/packages/sentinel_one/data_stream/alert/fields/agent.yml +++ b/packages/sentinel_one/data_stream/alert/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/sentinel_one/data_stream/alert/fields/ecs.yml b/packages/sentinel_one/data_stream/alert/fields/ecs.yml deleted file mode 100644 index 65edf4c29e3..00000000000 --- a/packages/sentinel_one/data_stream/alert/fields/ecs.yml +++ /dev/null @@ -1,116 +0,0 @@ -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: dll.hash.sha1 -- external: ecs - name: dll.path -- external: ecs - name: dns.question.name -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: file.created -- external: ecs - name: file.mtime -- external: ecs - name: network.direction -- external: ecs - name: observer.serial_number -- external: ecs - name: observer.version -- external: ecs - name: orchestrator.cluster.name -- external: ecs - name: orchestrator.namespace -- external: ecs - name: observer.os.name -- external: ecs - name: process.code_signature.signing_id -- external: ecs - name: process.command_line -- external: ecs - name: process.entity_id -- external: ecs - name: process.executable -- external: ecs - name: process.hash.md5 -- external: ecs - name: process.hash.sha1 -- external: ecs - name: process.hash.sha256 -- external: ecs - name: process.name -- external: ecs - name: process.parent.code_signature.signing_id -- external: ecs - name: process.parent.command_line -- external: ecs - name: process.parent.entity_id -- external: ecs - name: process.parent.executable -- external: ecs - name: process.parent.hash.md5 -- external: ecs - name: process.parent.hash.sha1 -- external: ecs - name: process.parent.hash.sha256 -- external: ecs - name: process.parent.name -- external: ecs - name: process.parent.pid -- external: ecs - name: process.parent.start -- external: ecs - name: process.parent.user.name -- external: ecs - name: process.pid -- external: ecs - name: process.start -- external: ecs - name: process.user.name -- external: ecs - name: registry.key -- external: ecs - name: registry.path -- external: ecs - name: registry.value -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: rule.category -- external: ecs - name: rule.description -- external: ecs - name: rule.id -- external: ecs - name: rule.name -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: tags -- external: ecs - name: user.domain -- external: ecs - name: user.name diff --git a/packages/sentinel_one/data_stream/group/fields/agent.yml b/packages/sentinel_one/data_stream/group/fields/agent.yml index 6e1bac042bc..894e6f12be2 100644 --- a/packages/sentinel_one/data_stream/group/fields/agent.yml +++ b/packages/sentinel_one/data_stream/group/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/sentinel_one/data_stream/group/fields/ecs.yml b/packages/sentinel_one/data_stream/group/fields/ecs.yml deleted file mode 100644 index 938c61044f1..00000000000 --- a/packages/sentinel_one/data_stream/group/fields/ecs.yml +++ /dev/null @@ -1,24 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: group.id -- external: ecs - name: group.name -- external: ecs - name: related.user -- external: ecs - name: tags -- external: ecs - name: user.full_name -- external: ecs - name: user.id diff --git a/packages/sentinel_one/data_stream/threat/fields/agent.yml b/packages/sentinel_one/data_stream/threat/fields/agent.yml index 6e1bac042bc..894e6f12be2 100644 --- a/packages/sentinel_one/data_stream/threat/fields/agent.yml +++ b/packages/sentinel_one/data_stream/threat/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/sentinel_one/data_stream/threat/fields/ecs.yml b/packages/sentinel_one/data_stream/threat/fields/ecs.yml deleted file mode 100644 index 97b9f837d5f..00000000000 --- a/packages/sentinel_one/data_stream/threat/fields/ecs.yml +++ /dev/null @@ -1,72 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.id -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: host.geo.city_name -- external: ecs - name: host.geo.continent_name -- external: ecs - name: host.geo.country_iso_code -- external: ecs - name: host.geo.country_name -- external: ecs - name: host.geo.location -- external: ecs - name: host.geo.region_iso_code -- external: ecs - name: host.geo.region_name -- external: ecs - name: host.os.type -- external: ecs - name: message -- external: ecs - name: observer.version -- external: ecs - name: process.name -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: tags -- external: ecs - name: threat.framework -- external: ecs - name: threat.indicator.file.extension -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.file.path -- external: ecs - name: threat.indicator.file.size -- external: ecs - name: threat.tactic.id -- external: ecs - name: threat.tactic.name -- external: ecs - name: threat.technique.id -- external: ecs - name: threat.technique.reference -- external: ecs - name: user.email -- external: ecs - name: user.name diff --git a/packages/sentinel_one/docs/README.md b/packages/sentinel_one/docs/README.md index f79898ca8cf..b62df0cb6ab 100644 --- a/packages/sentinel_one/docs/README.md +++ b/packages/sentinel_one/docs/README.md @@ -119,69 +119,18 @@ An example event for `activity` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.hash.sha1 | SHA1 hash. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.geo.city_name | City name. | keyword | -| host.geo.continent_name | Name of the continent. | keyword | -| host.geo.country_iso_code | Country ISO code. | keyword | -| host.geo.country_name | Country name. | keyword | -| host.geo.location | Longitude and latitude. | geo_point | -| host.geo.region_iso_code | Region ISO code. | keyword | -| host.geo.region_name | Region name. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| observer.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| observer.version | Observer version. | keyword | -| process.hash.sha1 | SHA1 hash. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | | sentinel_one.activity.account.id | Related account ID (if applicable). | keyword | | sentinel_one.activity.account.name | Related account name (if applicable). | keyword | | sentinel_one.activity.agent.id | Related agent (if applicable). | keyword | @@ -232,13 +181,6 @@ An example event for `activity` looks as following: | sentinel_one.activity.threat.id | Related threat ID (if applicable). | keyword | | sentinel_one.activity.type | Activity type. | long | | sentinel_one.activity.updated_at | Activity last updated time (UTC). | date | -| tags | List of keywords used to tag each event. | keyword | -| user.email | User email address. | keyword | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.group.id | Unique identifier for the group on the system/platform. | keyword | -| user.group.name | Name of the group. | keyword | -| user.id | Unique identifier of the user. | keyword | ### agent @@ -453,63 +395,17 @@ An example event for `agent` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| group.id | Unique identifier for the group on the system/platform. | keyword | -| group.name | Name of the group. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.geo.city_name | City name. | keyword | -| host.geo.continent_name | Name of the continent. | keyword | -| host.geo.country_iso_code | Country ISO code. | keyword | -| host.geo.country_name | Country name. | keyword | -| host.geo.location | Longitude and latitude. | geo_point | -| host.geo.region_iso_code | Region ISO code. | keyword | -| host.geo.region_name | Region name. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.type | Use the `os.type` field to categorize the operating system into one of the broad commercial families. If the OS you're dealing with is not listed as an expected value, the field should not be populated. Please let us know by opening an issue with ECS, to propose its addition. | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| observer.version | Observer version. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | | sentinel_one.agent.account.id | A reference to the containing account. | keyword | | sentinel_one.agent.account.name | Name of the containing account. | keyword | | sentinel_one.agent.active_directory.computer.member_of | Computer member of. | keyword | @@ -593,9 +489,6 @@ An example event for `agent` looks as following: | sentinel_one.agent.total_memory | Memory size (MB). | long | | sentinel_one.agent.user_action_needed | A list of pending user actions. | keyword | | sentinel_one.agent.uuid | Agent's universally unique identifier. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | ### alert @@ -886,106 +779,18 @@ An example event for `alert` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| dll.hash.sha1 | SHA1 hash. | keyword | -| dll.path | Full file path of the library. | keyword | -| dns.question.name | The name being queried. If the name field contains non-printable characters (below 32 or above 126), those characters should be represented as escaped base 10 integers (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, and line feeds should be converted to \t, \r, and \n respectively. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.created | File creation time. Note that not all filesystems store the creation time. | date | -| file.mtime | Last time the file content was modified. | date | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| observer.os.name | Operating system name, without the version. | keyword | -| observer.os.name.text | Multi-field of `observer.os.name`. | match_only_text | -| observer.serial_number | Observer serial number. | keyword | -| observer.version | Observer version. | keyword | -| orchestrator.cluster.name | Name of the cluster. | keyword | -| orchestrator.namespace | Namespace in which the action is taking place. | keyword | -| process.code_signature.signing_id | The identifier used to sign the process. This is used to identify the application manufactured by a software vendor. The field is relevant to Apple \*OS only. | keyword | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.executable | Absolute path to the process executable. | keyword | -| process.executable.text | Multi-field of `process.executable`. | match_only_text | -| process.hash.md5 | MD5 hash. | keyword | -| process.hash.sha1 | SHA1 hash. | keyword | -| process.hash.sha256 | SHA256 hash. | keyword | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.parent.code_signature.signing_id | The identifier used to sign the process. This is used to identify the application manufactured by a software vendor. The field is relevant to Apple \*OS only. | keyword | -| process.parent.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.parent.command_line.text | Multi-field of `process.parent.command_line`. | match_only_text | -| process.parent.entity_id | Unique identifier for the process. The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. | keyword | -| process.parent.executable | Absolute path to the process executable. | keyword | -| process.parent.executable.text | Multi-field of `process.parent.executable`. | match_only_text | -| process.parent.hash.md5 | MD5 hash. | keyword | -| process.parent.hash.sha1 | SHA1 hash. | keyword | -| process.parent.hash.sha256 | SHA256 hash. | keyword | -| process.parent.name | Process name. Sometimes called program name or similar. | keyword | -| process.parent.name.text | Multi-field of `process.parent.name`. | match_only_text | -| process.parent.pid | Process id. | long | -| process.parent.start | The time the process started. | date | -| process.parent.user.name | Short name or login of the user. | keyword | -| process.parent.user.name.text | Multi-field of `process.parent.user.name`. | match_only_text | -| process.pid | Process id. | long | -| process.start | The time the process started. | date | -| process.user.name | Short name or login of the user. | keyword | -| process.user.name.text | Multi-field of `process.user.name`. | match_only_text | -| registry.key | Hive-relative path of keys. | keyword | -| registry.path | Full path, including hive, key and value | keyword | -| registry.value | Name of the value written. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.category | A categorization value keyword used by the entity using the rule for detection of this event. | keyword | -| rule.description | The description of the rule generating the event. | keyword | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | | sentinel_one.alert.agent.site_id | Site id. | keyword | | sentinel_one.alert.analyst_verdict | Analyst verdict. | keyword | | sentinel_one.alert.container.info.labels | Container info labels. | keyword | @@ -1041,12 +846,6 @@ An example event for `alert` looks as following: | sentinel_one.alert.target.process.proc.storyline_id | Target Process StoryLine ID. | keyword | | sentinel_one.alert.target.process.proc.uid | Target Process Unique ID. | keyword | | sentinel_one.alert.target.process.start_time | Target Process Start Time. | date | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | ### group @@ -1138,53 +937,18 @@ An example event for `group` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| group.id | Unique identifier for the group on the system/platform. | keyword | -| group.name | Name of the group. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| related.user | All the user names or other user identifiers seen on the event. | keyword | | sentinel_one.group.agent.count | | long | | sentinel_one.group.created_at | | date | | sentinel_one.group.creator.id | | keyword | @@ -1196,10 +960,6 @@ An example event for `group` looks as following: | sentinel_one.group.registration_token | | keyword | | sentinel_one.group.site.id | | keyword | | sentinel_one.group.type | | keyword | -| tags | List of keywords used to tag each event. | keyword | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | ### threat @@ -1490,66 +1250,17 @@ An example event for `threat` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.geo.city_name | City name. | keyword | -| host.geo.continent_name | Name of the continent. | keyword | -| host.geo.country_iso_code | Country ISO code. | keyword | -| host.geo.country_name | Country name. | keyword | -| host.geo.location | Longitude and latitude. | geo_point | -| host.geo.region_iso_code | Region ISO code. | keyword | -| host.geo.region_name | Region name. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.type | Use the `os.type` field to categorize the operating system into one of the broad commercial families. If the OS you're dealing with is not listed as an expected value, the field should not be populated. Please let us know by opening an issue with ECS, to propose its addition. | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| observer.version | Observer version. | keyword | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | | sentinel_one.threat.agent.account.id | Account id. | keyword | | sentinel_one.threat.agent.account.name | Account name. | keyword | | sentinel_one.threat.agent.active_threats | Active threats. | long | @@ -1668,19 +1379,3 @@ An example event for `threat` looks as following: | sentinel_one.threat.storyline | Storyline identifier from agent. | keyword | | sentinel_one.threat.threat_id | Threat id. | keyword | | sentinel_one.threat.whitening_option | Whitening options. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| threat.framework | Name of the threat framework used to further categorize and classify the tactic and technique of the reported threat. Framework classification can be provided by detecting systems, evaluated at ingest time, or retrospectively tagged to events. | keyword | -| threat.indicator.file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| threat.indicator.file.path.text | Multi-field of `threat.indicator.file.path`. | match_only_text | -| threat.indicator.file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| threat.tactic.id | The id of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/ ) | keyword | -| threat.tactic.name | Name of the type of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/) | keyword | -| threat.technique.id | The id of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| threat.technique.reference | The reference url of technique used by this threat. You can use a MITRE ATT&CK® technique, for example. (ex. https://attack.mitre.org/techniques/T1059/) | keyword | -| user.email | User email address. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | diff --git a/packages/sentinel_one/manifest.yml b/packages/sentinel_one/manifest.yml index 07a356cddb0..103a8cb2d8c 100644 --- a/packages/sentinel_one/manifest.yml +++ b/packages/sentinel_one/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: sentinel_one title: SentinelOne -version: "1.23.3" +version: "1.24.0" description: Collect logs from SentinelOne with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - edr_xdr conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" screenshots: - src: /img/sentinel-one-screenshot.png title: SentinelOne Threat Dashboard Screenshot From 45d95640c9b03f9b7249f92ed2b441bdd9adfba6 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:49 +0930 Subject: [PATCH 082/121] [sentinel_one_cloud_funnel] - Updated fields definitions Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/sentinel_one_cloud_funnel --- packages/sentinel_one_cloud_funnel/changelog.yml | 5 +++++ .../data_stream/event/fields/beats.yml | 3 --- packages/sentinel_one_cloud_funnel/docs/README.md | 1 - packages/sentinel_one_cloud_funnel/manifest.yml | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/sentinel_one_cloud_funnel/changelog.yml b/packages/sentinel_one_cloud_funnel/changelog.yml index 8155861042b..ac5ef5e71c2 100644 --- a/packages/sentinel_one_cloud_funnel/changelog.yml +++ b/packages/sentinel_one_cloud_funnel/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.2.0" + changes: + - description: Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.1.0" changes: - description: Improve `dns fields` to process `dns.answers.type` and `dns.questions.type`. diff --git a/packages/sentinel_one_cloud_funnel/data_stream/event/fields/beats.yml b/packages/sentinel_one_cloud_funnel/data_stream/event/fields/beats.yml index 0657078efcf..b9a19f1aa20 100644 --- a/packages/sentinel_one_cloud_funnel/data_stream/event/fields/beats.yml +++ b/packages/sentinel_one_cloud_funnel/data_stream/event/fields/beats.yml @@ -4,9 +4,6 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. - name: aws.s3 type: group fields: diff --git a/packages/sentinel_one_cloud_funnel/docs/README.md b/packages/sentinel_one_cloud_funnel/docs/README.md index addd1fe62b9..74ed92a1074 100644 --- a/packages/sentinel_one_cloud_funnel/docs/README.md +++ b/packages/sentinel_one_cloud_funnel/docs/README.md @@ -887,6 +887,5 @@ An example event for `event` looks as following: | sentinel_one_cloud_funnel.event.url.action | URL action of process. | keyword | | sentinel_one_cloud_funnel.event.url.address | Complete URL. | keyword | | sentinel_one_cloud_funnel.event.url.source | | keyword | -| tags | User defined tags. | keyword | diff --git a/packages/sentinel_one_cloud_funnel/manifest.yml b/packages/sentinel_one_cloud_funnel/manifest.yml index 149576b63dd..256d4fa2631 100644 --- a/packages/sentinel_one_cloud_funnel/manifest.yml +++ b/packages/sentinel_one_cloud_funnel/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: sentinel_one_cloud_funnel title: SentinelOne Cloud Funnel -version: "1.1.0" +version: "1.2.0" description: Collect logs from SentinelOne Cloud Funnel with Elastic Agent. type: integration categories: ["security", "edr_xdr"] From a2379f4b32954dbed6ef896f273d1f8d88b25944 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:50 +0930 Subject: [PATCH 083/121] [slack] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/slack --- packages/slack/changelog.yml | 5 ++ .../slack/data_stream/audit/fields/agent.yml | 50 ------------- .../slack/data_stream/audit/fields/beats.yml | 3 - .../slack/data_stream/audit/fields/ecs.yml | 72 ------------------- packages/slack/docs/README.md | 68 ------------------ packages/slack/manifest.yml | 4 +- 6 files changed, 7 insertions(+), 195 deletions(-) delete mode 100644 packages/slack/data_stream/audit/fields/ecs.yml diff --git a/packages/slack/changelog.yml b/packages/slack/changelog.yml index 5638392b362..b8a54edc9bb 100644 --- a/packages/slack/changelog.yml +++ b/packages/slack/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.21.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.20.0" changes: - description: Improve handling of empty responses. diff --git a/packages/slack/data_stream/audit/fields/agent.yml b/packages/slack/data_stream/audit/fields/agent.yml index bca66ea4ae0..4b15225a4d4 100644 --- a/packages/slack/data_stream/audit/fields/agent.yml +++ b/packages/slack/data_stream/audit/fields/agent.yml @@ -1,56 +1,6 @@ -- name: cloud.account.id - external: ecs -- name: cloud.availability_zone - external: ecs -- name: cloud.instance.id - external: ecs -- name: cloud.instance.name - external: ecs -- name: cloud.machine.type - external: ecs -- name: cloud.provider - external: ecs -- name: cloud.region - external: ecs -- name: cloud.project.id - external: ecs - name: cloud.image.id type: keyword description: Image ID for the cloud instance. -- name: container.id - external: ecs -- name: container.image.name - external: ecs -- name: container.labels - external: ecs -- name: container.name - external: ecs -- name: host.architecture - external: ecs -- name: host.domain - external: ecs -- name: host.hostname - external: ecs -- name: host.id - external: ecs -- name: host.ip - external: ecs -- name: host.mac - external: ecs -- name: host.name - external: ecs -- name: host.os.family - external: ecs -- name: host.os.kernel - external: ecs -- name: host.os.name - external: ecs -- name: host.os.platform - external: ecs -- name: host.os.version - external: ecs -- name: host.type - external: ecs - name: host.containerized type: boolean description: If the host is a container. diff --git a/packages/slack/data_stream/audit/fields/beats.yml b/packages/slack/data_stream/audit/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/slack/data_stream/audit/fields/beats.yml +++ b/packages/slack/data_stream/audit/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/slack/data_stream/audit/fields/ecs.yml b/packages/slack/data_stream/audit/fields/ecs.yml deleted file mode 100644 index f7210a05058..00000000000 --- a/packages/slack/data_stream/audit/fields/ecs.yml +++ /dev/null @@ -1,72 +0,0 @@ -- name: ecs.version - external: ecs -- name: error.message - external: ecs -- name: event.action - external: ecs -- name: event.id - external: ecs -- name: event.type - external: ecs -- name: event.category - external: ecs -- name: event.kind - external: ecs -- name: event.ingested - external: ecs -- name: event.original - external: ecs -- name: file.hash.md5 - external: ecs -- name: related.ip - external: ecs -- name: related.user - external: ecs -- name: source.address - external: ecs -- name: source.as.number - external: ecs -- name: source.as.organization.name - external: ecs -- name: source.geo.city_name - external: ecs -- name: source.geo.continent_name - external: ecs -- name: source.geo.country_iso_code - external: ecs -- name: source.geo.country_name - external: ecs -- name: source.geo.location - external: ecs -- name: source.geo.name - external: ecs -- name: source.geo.region_iso_code - external: ecs -- name: source.geo.region_name - external: ecs -- name: source.ip - external: ecs -- name: related.hash - external: ecs -- name: tags - external: ecs -- name: user.email - external: ecs -- name: user.id - external: ecs -- name: user.full_name - external: ecs -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version diff --git a/packages/slack/docs/README.md b/packages/slack/docs/README.md index d781a6250fd..04cc62894d4 100644 --- a/packages/slack/docs/README.md +++ b/packages/slack/docs/README.md @@ -40,58 +40,18 @@ Audit logs summarize the history of changes made within the Slack Enterprise. | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | | event.dataset | Event dataset | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.hash.md5 | MD5 hash. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | | slack.audit.context.domain | The domain of the Workspace or Enterprise | keyword | | slack.audit.context.id | The ID of the workspace or enterprise | keyword | | slack.audit.context.name | The name of the workspace or enterprise | keyword | @@ -123,34 +83,6 @@ Audit logs summarize the history of changes made within the Slack Enterprise. | slack.audit.entity.timestamp | The timestamp of the entity when entity_type is message | keyword | | slack.audit.entity.title | Title of the entity when entity_type is file | keyword | | slack.audit.entity.type | The type of the entity when entity_type is role | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| user.email | User email address. | keyword | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | An example event for `audit` looks as following: diff --git a/packages/slack/manifest.yml b/packages/slack/manifest.yml index 493c3b8c4fb..42fc10f6db4 100644 --- a/packages/slack/manifest.yml +++ b/packages/slack/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: slack title: "Slack Logs" -version: "1.20.0" +version: "1.21.0" description: "Slack Logs Integration" type: integration categories: @@ -9,7 +9,7 @@ categories: - security conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" icons: - src: /img/slack.svg title: Slack logo From 10249049804c15408a48af4b7a6b48fc287703b0 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:53 +0930 Subject: [PATCH 084/121] [snyk] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. The set ecs.version processor in pipelines was changed 8.11.0. Previously the pipeline was setting version 8.12.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. The ecs.version in sample_event.json files was changed to 8.11.0. Previously sample_event.json files contained 8.12.0. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/snyk --- .../snyk/_dev/deploy/docker/files/config.yml | 2 - packages/snyk/changelog.yml | 5 + .../snyk/data_stream/audit/fields/agent.yml | 93 +------------ .../snyk/data_stream/audit/fields/beats.yml | 3 - .../snyk/data_stream/audit/fields/ecs.yml | 14 -- .../_dev/test/pipeline/test-snyk-audit.json | 124 +++++++++--------- .../test-snyk-audit.json-expected.json | 40 +++--- .../elasticsearch/ingest_pipeline/default.yml | 2 +- .../data_stream/audit_logs/fields/agent.yml | 93 +------------ .../data_stream/audit_logs/fields/beats.yml | 3 - .../data_stream/audit_logs/fields/ecs.yml | 28 ---- .../data_stream/audit_logs/sample_event.json | 2 +- .../snyk/data_stream/issues/fields/agent.yml | 93 +------------ .../snyk/data_stream/issues/fields/beats.yml | 3 - .../snyk/data_stream/issues/fields/ecs.yml | 34 ----- .../vulnerabilities/fields/agent.yml | 93 +------------ .../vulnerabilities/fields/beats.yml | 3 - .../vulnerabilities/fields/ecs.yml | 32 ----- packages/snyk/docs/README.md | 117 +---------------- packages/snyk/manifest.yml | 4 +- 20 files changed, 96 insertions(+), 692 deletions(-) delete mode 100644 packages/snyk/data_stream/audit/fields/ecs.yml delete mode 100644 packages/snyk/data_stream/audit_logs/fields/ecs.yml delete mode 100644 packages/snyk/data_stream/issues/fields/ecs.yml delete mode 100644 packages/snyk/data_stream/vulnerabilities/fields/ecs.yml diff --git a/packages/snyk/_dev/deploy/docker/files/config.yml b/packages/snyk/_dev/deploy/docker/files/config.yml index 9d5cfc86cd0..d3ac62e9a10 100644 --- a/packages/snyk/_dev/deploy/docker/files/config.yml +++ b/packages/snyk/_dev/deploy/docker/files/config.yml @@ -73,7 +73,6 @@ rules: {"issue":{"url":"https://snyk.io/vuln/SNYK-GOLANG-GITHUBCOMGOYAMLYAML-564236","id":"SNYK-GOLANG-GITHUBCOMGOYAMLYAML-564236","title":"Denial of Service (DoS)","type":"vuln","package":"github.com/go-yaml/yaml","version":"2.1.0","severity":"medium","originalSeverity":null,"uniqueSeveritiesList":["medium"],"language":"golang","packageManager":"golang","semver":{"vulnerable":["<2.2.8"],"hashesRange":["53403b58ad1b561927d19068c655246f2db79d48"],"vulnerableHashes":["dd8f49ae7840d1fc6810d53ee7b05356da92f81f","d4766d1dff71f8a135a57e1fcff946c8c1a140ab","2aba0a492be00f1eb4d95483b08930ebe4968b64","3b0eedc5a476efc2b2e025eff55b2fd08fa32abd","2f2fd02e5a54a7d4f5e5d3494b170b0cb9275c92","7ad95dd0798a40da1ccdff6dff35fd177b5edf40","f7716cbe52baa25d2e9b0d0da546fcf909fc16b4","1ff37a7d30b085dc643dee7adb18759e3511661a","eca94c41d994ae2215d455ce578ae6e2dc6ee516","b0c168ac0cf9493da1f9bb76c34b26ffef940b4a","77373ee937410eceadc4dc64b1100d897ed593d0","025607cd2e381e6e08a56ffec46ac79e23ca2d88","7d17c9173a3d25ebba15cedb25b5205bdfb1eac8","ca3d523f32f3b33fb3265bfeb8e11003a8670e3d","85db785e81ed62ffae7a145404fc0f022335378c","a72a87d92dad7563e31c2c007e8d67f93d67f221","1be3d31502d6eabc0dd7ce5b0daab022e14a5538","90376f16b6d74c4e2fff21dd24397bec3dc62dd5","bb263360b83253468e534d974aabeddd6c22f887","d466437aa4adc35830964cffc5b5f262c63ddcb4","d6c23fbaf16f72995b58492627e65801cfb9a8dd","e4d366fc3c7938e2958e662b4258c7a89e1f0e3e","60a2abf4e00318875a661c29b36df7a68e484bf4","f4d271a8a289b41fa88b802c430fefde4e018bba","10c59a7d91867c206737dcd482fe68906a1484ca","d0b6f3facf302fb1bf969a12bad68ce720b3c025","4d6bb54d8acc91e147763cea066cff0b89437e90","1244d3ce02e3e1c16820ada0bae506b6c479f106","49fdd64ad429d146bacf7106dd73078e889be2e8","8e626dec39b5836cef636d885e33479debcf0cb1","4914593b9558e85597f08346c798aea8f6fb899f","031c922227a592b2b562a1833438308381f9a8bf","b51f82a2e3cbedab685908bd64d61d0a1b781754","c75e52ecee48db6de9aa73d00a360d43abf3e7ac","857a0b2759f87f47aaebad6dd319cf4f887eb6dc","5887bc194be84805c8283e9d9a66102bf9571fca","a528d0ef484d32e416d7b9c4a249d1fa7111be6e","5b18502a28c65dfd209ea5aebb405fb6fc07f7e1","5d6f7e02b7cdad63b06ab3877915532cd30073b4","9c272e25743608d6d3287141522eb4506b2dac45","125a562d7bf105e062ed2adfb2d37e6f11c209bd","87e4a22b684220ccca96de3f2e651b2380a55f9e","d56ec34a3ded0bb58c82198664664ccb81eec91b","b754a4fe6ad8db932e083a2d85ae2199b3516bef","04092268b2c5e87e6373229049c827b833af4edb","f59f5e67022f3c186e20af01b1993b86ac74f0dc","52d5976e4791cf8c96a9de7569098e3752677412","770b8dae4cf00919e5eafffbd8d58186294b61b5","71e7ede9d48a2e096f6d5d0516c763513a471bd1","b01920c75e30179201b01633db246038b0226ce9","ef0aede23c8c624e127a9a59183ee8915e48a3c9","1632dd8118ce1efece66b7f53bb167956d5d8b4e","05299e459464264cd87a230b62d1aca93725c51b","d00346f943c9d2c43424c8a3840f5ca58817750d","49c95bdc21843256fb6c4e0d370a05f24a0bf213","088598405c86d37e951287d094d691e221654a00","c11897f0ba79d8a35d8a124ff0d76e13d9dccb9b","711419034010345c604724ef87ec3db91ffe0936","3e6d767784b037b90a14701b6c9f0643f05db963","a83829b6f1293c91addabc89d0571c246397bbf4","ee2f4956ea46791a74a31142105f03c0d5f9492b","7b079234548be56f14c6e342d4660aa8d54865b7","b7fbda9990042cd5456fdf187480c25fdd776f92","a6dc653f939ab0e6a554873806c41add1140d90c","687eda924018599a7c4518013c369f0bfb7eb0e1","fa9662d290d59b79f2ef7e1f72c885560efe512d","e47eca576e8f3a433de0ba77f1923e7c7f959667","e90bcf783f7abddaa0ee0994a09e536498744e49","fdc1ab46101a842d9e914408bd481f6647d5f9c1","f0766b44ca7999dc9af38a050ddf6db79d05bf3b","cdd36ee8d333aa740c1c0bceae0da74969b2c60b","7701d177ce02b7bd38c4ebd2ba4a7783080505ae","2c1be0d7f7ff8305cf666e89152e9753c8b39004","97203c6e4fc7347bfef3bd6d4913e90bd46c7ecb","7c97801ccf41d5273de9e22c8b2af6860c7703a2","7002636de42c9ef59a2921bb4f78744cabe8bfe3","0725b7707fdeeb6894c403d0f5a2a20e1dc7454d","1dd72ac3928693b9db2533639dfc2a5f831697eb","73a1567027eea2fab2b057a193036f844736f7da","7539b1dee2c790ab2d1aa5e254ef877f5552ff97","920b7d819b42f26f4796e4a43f518090a7a6331f","1f64d6156d11335c3f22d9330b0ad14fc1e789ce","1b9791953ba4027efaeb728c7355e542a203be5e","1ed59511881fdb008c1e618e9f219ce0704e658e","c325d146e464fb9567e780ddfa2dad3a99323075","0ee36981cbf495d5eb6aeb540a3afc25c61d1a96","c4a9fb418357aceb801272d73efd518f183700fa","a347d2466e459933f4fb25f8026d995977436ccf","f221b8435cfb71e54062f6c6e99e9ade30b124d5","5206f6dd03423b3a5462a2a4286a4efae8abe347","a1c4bcb6c278a41992e2f4f0f29a44b4146daa5c","4ca689e686c2caf4dda3a62936c097d6dfb56877","119a11e4378a0410c69c42d82f51331a6da7a97c","c7da9dcff86f24fcfdc15e1f9fa39dfc19784616","f29dde21846f6357ee4421013b59eefd65c069b0","5515099aacaeb9ff3ab7492f0803327bb19fc512","1c9241b56a03383c77e1c33d86ea6ca4a927153e","86f5ed62f8a0ee96bd888d2efdfd6d4fb100a4eb","1f2a25ba9402c70a7806e84531ef763943739072","1418a9bc452f9cf4efa70307cafcb10743e64a56","65b1927d8262617ca3d25f296fdde1e8c48f813d","2bf60357b89cbc6044dde700cf63bab94a615bf7","c6314f5b627e2a1c1846d89cd775de6b2808d37e","50e1b1b1332ea40fff2a9b13bfbccbbecd526f00","50f7813e6b19e58334360ab011dfbaece5b1501f","a311394a2a9276454d3f92d26838c3ae3d99cdf3","79f5ef7c40ae7a4ee6bcd26d324bf50491b431e5","731788bc8b082f8c81c63ca0abd5950c7a68a2f1","6491ec31f7b0d27492e3046c86de94838dcb523c","41168bb7ed2fc849bc36727a2b902bd8f447bfc2","bc27649cd5454055cf20fdb9ef556c214d3f9aa0","d6b53382672776035ad8ef0404681f8a4a16bb95","8eba062837dc10754db7cbafcbedbfbc985ca172","837b0877fcd6b2c8ba83d126917267695ff16ad8","72c33f6840f49f9ed7d1faef7562b3266640fdf4","26b882523374125854702734c30b0ce6a1a18d7b","e90048704a8adb0b81b2e15ebafd1a35fa110903","4fc5987536ef307a24ca299aee7ae301cde3d221","4341420a144323d3f148ece677a20da6e077cfd2","5c8bfe59213b6e9a5eb50debebc396e99a9fa174","200c098a06472243b50aeda4510220a90c4e7dbe","de3643d77b438c6f0f69f350c437639a300b5e73","9a4310b1caff4cca3780580195a916ca060d08f7","91eb945ac02153399ac9f69e34751f1a176254c3","4cdd993908b57c3b87bef0695e5ca989151ad55f","7ddc4634ce2d8ca5c03846918ae1df6aa40ee464","ec232d2920a84930b077414b60b5985e076ae228","2c8612dfee1362e7e482c66c5feb892a94d53255","d670f9405373e636a5a2765eea47fac0c9bc91a4","e9bfed595636e952566e5cb857c22b918f2530a2","c1cd2254a6dd314c9d73c338c12688c9325d85c6","df747160af0ebfcc572951e4168d4b1bc91a47f5","a65e08b08285cef29253c50ffd92469bf6e26a29","e6da37e746419537560c1e95e429f42b33f6d0e3","eea198a9c5cc6e02bfcd130a932051088a9f0950","6675ed2a9028caf87bb5915503c08a595e57b77d","562080bfe963d41a6870a4c500918f6361a0b61f","8171f560dedcb162dd3d2c925015679e84bac269","c78cd3ebd83777ac093137fbb55c33a9d3f65819","e4ac4c457c23b390e7fd75ddf746c5a69aa8cfd5","93d787c44dc828e1c67fa275cb66eb86bb2929f8","7cdd87a79f79db641dae55776224443026d28928","406cad6bb47dd7d9a123d005fb8ff766f6463051","523c7d9470684b02d902e8d986cd9eea66884755","9ca8abd6882a6e741166e6ec946a73f3a64df65a","885e19c0dda1f4e4e22837474879f8f3d36fb449","e8976af76e3d35c48f8b2c9540cca3e92995fbc6","addb3a024ff5763c8facbe4767fe530d602cfedc","c7f6f9c6e6c14027a46eb91241427dba67604f39","0a6d1b02c16e372ceea8f17f3b1833b918954bf1","835086a6b6aa65939515e30b5d6c2eba43d7c075","7b8fd2dbef04521fdd8d670ef4c77be691845aa2","3eb2270747cdd89e3f095cb24e8dd4ccf2a098f6","1d653a737648051ca638423377052c2f5c10c050","14d1c4659ec7b9ee26f5d705f3c2bb56cb6cbee4","c544d0342172409bd9c8f7c45d9fb21971c8aee9","6941443daa441371720e9ef8f3554c3958cfb071","f8db564a0a4a5f6d04f66522493597f18e5ab4ae","7c634f6a68c1076d3cfdc56930db26e86f7876d7","f7e23311052d3dda728ce15788fb3727898afa17","8691640bc70f3d96128a809341d850b550a3abb9","b9b22c434500d7639936fbed673fc0ef23ce88f6","d6385b38675d8d03521c9290f4f3d7bff08664c0","4c78c975fe7c825c6d1466c42be594d1d6f3aba6","54c736c86c9bcc793fb4bd6f203604cd738dc0e9","722ff6b958a31d4ca3405db35a72648a6077a6bb","2afc2e57e051513a3f5f67e74857696a8558d67b","283fbcdd1e64975730a38609f8802ef983a43cb9","ab5d55c35f3919fe06e9daedce5a32f4aab23777","e2fbf5b72a6a12abd15be9b37656a0a136fc32f8","399c3345e0f76f583d830cd7da27518bbb00c91a","b6679148d27038e59d7818facc4d100e677a64ae","43a0256bb22b0c2e1803ac6e28f55e5989a60523","f5f5cc19d1f681884684426c96adadef47a3b55c","787afde64d7b36591050440c4a14c2288b373de6","7b8349ac747c6a24702b762d2c4fd9266cf4f1d6","0e4404da71227dcc02fb1deee803d93e86d08f72","a95acef3719e5e9f7614cc90a119dee4699291eb","3ba0e99ffa727bd7eb782b7a5d1aafcb989b0899","5edc3ded41385ca1b9a80339d2a070e4d0a17cb6","2c9db3558be789ef3896b03ed3f354b822c304b9","a833012353d046b1f12c82db87d01c86570b24d7","77b516425597da3c093a666c11608112e91604de","1ade51a028efa6990b524e0b01237dbd9123957d","9e27074feeaed4b0ae4e5e71187eff80c0f0bf35","cd515839285fe1a31b92193360172d59f818c9b8","9f33a69b86c3c76c52e41d12d83e233065bfcca9","36babc3691687601732d9e2571b698be4116469a","51d6538a90f86fe93ac480b35f37b2be17fef232","31c299268d302dd0aa9a0dcf765a3d58971ac83f","3e92d6a11b92fa4612d66712704844bdc0c48aed","9211cbc02789a32acf5e90c23a42f040ac3ec3f8","0cb32393ebcfc65467398e5daadfb63b2184caea","0f9a5c380d77a8b2888a78c3d3a14db15949b1fa","82377a97b299347cd15cc1be13e1c8d04e33efbb","fe9486c37432968838e1798b2317dc1aa10b586b","77b384eced7745af978888311ea3c67e57c7ed96","fc7f19eff1782a0beae3065097c776183e7d01d0","dbd6d0229d1f1e1c3055cd82efb81f60a27d1103","25c4ec802a7d637f88d584ab26798e94ad14c13b","5e76f7cf8cb1fc353b84b96c72a36c4984cbd005","a5844a8f8f489bad96ab6da62cfa21ee1f5d9e6b","41c132e8ac051886e4eb06e7c3d58ced63d58057","4f03e946c120a8f146f43bee6f392f9bb5d0a677","287cf08546ab5e7e37d55a84f7ed3fd1db036de5","1092c5d94f266e0f94e485a24f7010da877eeba0","910de082618d0d8ccac6443a6e7a72cc8bcd5227","feb4ca79644e8e7e39c06095246ee54b1282c118","3c68098bffba683534584be69216dac3a2b2305a","3323b7713e656f16fbd0eec27c60370b6237f4e3","f3293401ceedf2a32a1c22cb062b274dba6be798","43607cc2a1772b23faf366c24b8e33541187b64d","add015b1c64e144664b73d5eacfeb6aeace2e45c","3e69410288aeb97d31353af8e063b798d40feb3f","39e59aa7e15898a87148f0f4891a085c83b9b0fc","a3f3340b5840cee44f372bddb5880fcbc419b46a","05d405925260878bd750ea7d96c746c2d726b349","65622dcbf4c25328cd440d1b322c6530abe83337","8ca81d591dc2242f9c4b7a907533f0b7f93802b5","3d8cfc3754fba03b8f1a0d44ea4e6e870cf86c57","eb3733d160e74a9c7e442f435eb3bea458e1d19f","d0fefed9b627fbe0c1597ac29ed5f48ff2eb9064","dcd83b31fd165d8cc8677fce58f889dca3e06f35","7f97868eec74b32b0982dd158a51a446d1da7eb5","925f818e2c358746b3a14bf3e5614db14208037f","c95af922eae69f190717a0b7148960af8c55a072","0516c53462e633a479f3826e1d3557033413eeb8","53087c11c10b453af4f2eb47471434eae75526f9","5420a8b6744d3b0345ab293f6fcba19c978f1183","fb03f24d58ac0c7a3d85edc1b91dfcfea4329883","08434a82b8376f585898a97654ce18065d14cb97","a5b47d31c556af34a302ce5d659e6fea44d90de0","838f4ea96166350b9185bf3d2cbf786d34127ca2","f2d2788ce5b1741745c0d1a853e856b5b77376b2","284796d39ddb313ec0ae04898de280d41fe32479","970885f01c8bc1fecb7ab1c8ce8e7609bda45530","4f3d34e492b8930c50204a216d960e7da0dc5f63","9f389a1f0b1d442eba00213e7aa09ccd878d18b0","1b2e8c1531abbfe7dcd3de8ff4483326af275bc8","14227de293ca979cf205cd88769fe71ed96a97e2","e72f93569ef83aca933836c2fb9185faeeced236","3b4ad1db5b2a649883ff3782f5f9f6fb52be71af","a0ae8d516398f3724bb3db614ab47f0e4f643f2e","f7a330473f18ddc052fce1f71a2b2d1231860f71","81205292aba40f8868069e2f18d90043d3e724a6","059398de19c863a04c55315526d6c226de540aa1","e6ec13e5a80029d7ebcbc2c90d16ce5ff1fa6c84","8173ecbc8953a159ae0fa2fad94adf3553b0bf8e","b7dfe2d918fda477aa5b42519294b5ada3c991fa","b6b591a3c0ec0452719f4d4555a3e084fd9f12fb","ba29208cca8f239f2cea685183f79df8e4defc29","422f540d2e1f1b41b6184903cd1eb69c777df1bb","914e67f109a574665d15c0d179cdc796abefb176","1bf6a7ce154075e61134f8a68dd50902c3027a10","2628b30e544c309ac3d0c8cd7e78a785400cd41f","0846a25da24891a7b3c725bc190493b5f7525db8","4cadac2bc790baeffa0a7fa19689223966a64c24","b3031338ac8e006cbd668f67c36c24d2c5e64b6d","cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b","205b70273c7999d96b32db43ab54337690817184","62e345dcf33dd13810ceba10407c30a7db6a0958","53feefa2559fb8dfa8d81baad31be332c97d6c77","e720624475f3807e3dc6477e7af6feb09da0b848","bd61a856f807e525beaee41959452c88c83d46cf","f90ceb4f409096b60e2e9076b38b304b8246e5fa","3c0d4d4f56c36fcfd2da00ff26c40046512b4208","1f1f61830e4c9f1eff03047c9d1d11e576853bc4","f96735bc0fa70a12e9f41277b2d909e0c477ee30","e334f8522ac9fe2b381c329b3159a328eeb14f76","18e5f12b39cb93b31a249fb7115b9bbf6162aeeb","b3472531944cd769419f297322dc285a0fc0d6cc","3e542fbf7c84c0bf22f51ad07899cf80f8658caa","00efe9c47819ca58089c4bd5d1d8463248e23228","670d4cfef0544295bc27a114dbac37980d83185a","8ed39f36d6f36299d2ce5f9b35a05d048500f777","bb4e33bf68bf89cad44d386192cbed201f35b241","bef53efd0c76e49e6de55ead051f886bea7e9420","9eade332f0ceebc6b7c9e24893574cad4c51722b"]},"isIgnored":false,"publicationTime":"2020-04-02T11:29:49.000Z","disclosureTime":"2020-03-26T11:30:05.000Z","isUpgradable":false,"isPatchable":false,"isPinnable":false,"identifiers":{"CVE":["CVE-2019-11254"],"CWE":["CWE-1050"]},"credit":["Unknown"],"CVSSv3":"CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:N/I:N/A:H","cvssScore":"6.5","patches":[],"isPatched":false,"exploitMaturity":"no-known-exploit","reachability":"No Info","priorityScore":325,"jiraIssueUrl":null},"isFixed":false,"introducedDate":"2020-04-29","projects":[{"url":"https://snyk.io/org/orgname/project/projectid","id":"projectid","name":"username/reponame","source":"github","packageManager":"npm","targetFile":"package.json"},{"url":"https://snyk.io/org/orgname/project/projectid","id":"projectid","name":"someotheruser/someotherreponame","source":"github","packageManager":"npm","targetFile":"folder1/package.json"},{"url":"https://snyk.io/org/orgname/project/projectid","id":"projectid","name":"projectname","source":"cli","packageManager":"npm","targetFile":"package.json"}]} ] } - - path: /rest/orgs/0de7b2d6-c1da-46aa-887e-1886f96770d4/audit_logs/search methods: ["GET"] request_headers: @@ -276,7 +275,6 @@ rules: } } `}} - - path: /rest/orgs/0de7b2d6-c1da-46aa-887e-1886f96770d4/issues methods: ["GET"] request_headers: diff --git a/packages/snyk/changelog.yml b/packages/snyk/changelog.yml index 4eae76aa3d8..fdbc1797dcf 100644 --- a/packages/snyk/changelog.yml +++ b/packages/snyk/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.23.0" + changes: + - description: ECS version updated to 8.11.0. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.22.1" changes: - description: Fix handling of event filter parameter in audit_logs data stream. diff --git a/packages/snyk/data_stream/audit/fields/agent.yml b/packages/snyk/data_stream/audit/fields/agent.yml index 4d9a6f7b362..bc42d0a853b 100644 --- a/packages/snyk/data_stream/audit/fields/agent.yml +++ b/packages/snyk/data_stream/audit/fields/agent.yml @@ -1,100 +1,9 @@ - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/snyk/data_stream/audit/fields/beats.yml b/packages/snyk/data_stream/audit/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/snyk/data_stream/audit/fields/beats.yml +++ b/packages/snyk/data_stream/audit/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/snyk/data_stream/audit/fields/ecs.yml b/packages/snyk/data_stream/audit/fields/ecs.yml deleted file mode 100644 index f243288cc96..00000000000 --- a/packages/snyk/data_stream/audit/fields/ecs.yml +++ /dev/null @@ -1,14 +0,0 @@ -- name: event.created - external: ecs -- name: event.original - external: ecs -- name: tags - external: ecs -- name: message - external: ecs -- name: ecs.version - external: ecs -- name: user.group.id - external: ecs -- name: user.id - external: ecs diff --git a/packages/snyk/data_stream/audit_logs/_dev/test/pipeline/test-snyk-audit.json b/packages/snyk/data_stream/audit_logs/_dev/test/pipeline/test-snyk-audit.json index 2718a62df19..3c735093d02 100644 --- a/packages/snyk/data_stream/audit_logs/_dev/test/pipeline/test-snyk-audit.json +++ b/packages/snyk/data_stream/audit_logs/_dev/test/pipeline/test-snyk-audit.json @@ -1,64 +1,64 @@ { - "events": [ - { - "message": "{\"content\":{\"after\":{\"name\":\"elastic-integration\"},\"before\":{\"name\":\"admin.user\"}},\"created\":\"2024-04-15T19:47:21.565Z\",\"event\":\"org.edit\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"user_id\":\"b4b324c4-a55c-4cd6-82b8-f96e3b3b8d85\"}" - }, - { - "message": "{\"content\":{\"email\":\"other.user@company.com\",\"role\":\"0afa84b4-9d99-4c2b-94c2-65da22270836\"},\"created\":\"2024-04-15T19:49:01.920Z\",\"event\":\"org.user.invite\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"user_id\":\"b4b324c4-a55c-4cd6-82b8-f96e3b3b8d85\"}" - }, - { - "message": "{\"content\":{\"openInvitePublicId\":\"5cc83241-4d28-401e-80e7-3a421cee2c03\",\"url\":\"https://app.snyk.io/invite/link/accept?invite=5cc83241-4d28-401e-80e7-3a421cee2c03\\u0026utm_source=link_invite\\u0026utm_medium=referral\\u0026utm_campaign=product-link-invite\\u0026from=link_invite\"},\"created\":\"2024-04-16T09:46:29.448Z\",\"event\":\"org.user.invite_link.create\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"user_id\":\"b4b324c4-a55c-4cd6-82b8-f96e3b3b8d85\"}" - }, - { - "message": "{\"content\":{\"role\":\"ADMIN\",\"rolePublicId\":\"0afa84b4-9d99-4c2b-94c2-65da22270836\",\"userPublicId\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"},\"created\":\"2024-04-16T21:54:33.257Z\",\"event\":\"org.user.add\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" - }, - { - "message": "{\"content\":{\"email\":\"other.user-alias@company.com\",\"invitingUserId\":1881478,\"rolePublicId\":\"0afa84b4-9d99-4c2b-94c2-65da22270836\"},\"created\":\"2024-04-16T21:54:33.257Z\",\"event\":\"org.user.invite_link.accept\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" - }, - { - "message": "{\"content\":{\"origin\":\"cli\"},\"created\":\"2024-04-17T01:08:08.228Z\",\"event\":\"org.project.test\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" - }, - { - "message": "{\"content\":{\"after\":{\"autoDepUpgradeEnabled\":true,\"autoRemediationPrs\":{\"backlogPrStrategy\":\"vuln\",\"container\":{\"enabled\":true},\"freshPrsEnabled\":true,\"usePatchRemediation\":true},\"pullRequestTestEnabled\":true,\"reachableVulns\":{}},\"before\":{\"autoDepUpgradeEnabled\":true,\"autoRemediationPrs\":{\"backlogPrStrategy\":\"vuln\",\"container\":{},\"freshPrsEnabled\":true,\"usePatchRemediation\":true},\"pullRequestTestEnabled\":true,\"reachableVulns\":{}},\"flow\":\"on-boarding\",\"sourceType\":\"github\"},\"created\":\"2024-04-17T01:24:49.748Z\",\"event\":\"org.integration.settings.edit\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" - }, - { - "message": "{\"content\":{\"after\":{\"sastSettings\":{\"sastEnabled\":true}},\"before\":{\"sastSettings\":{}},\"interface\":\"ui\"},\"created\":\"2024-04-17T01:24:49.837Z\",\"event\":\"org.sast_settings.edit\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" - }, - { - "message": "{\"content\":{\"targetId\":\"693b1550-43b1-4108-a55a-37e5cabe7355\"},\"created\":\"2024-04-17T01:26:10.024Z\",\"event\":\"org.target.create\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" - }, - { - "message": "{\"content\":{\"action\":\"Cloned repo: https://github.com/elastic/mito.git commit hash: 5e8963319b4a55b32f6d9db0a19f0ddd70ae8c5d\",\"requestId\":\"dc3cad34-9ece-49d6-b38e-28d8439f29e1\"},\"created\":\"2024-04-17T01:26:13.000Z\",\"event\":\"org.project.files.create\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\"}" - }, - { - "message": "{\"content\":{\"AboveSizeLimit\":{},\"action\":\"Modify files - exclude\",\"excluded\":{},\"notSupported\":{\"\":1,\".cel\":1,\".json\":1,\".md\":1,\".mod\":1,\".sum\":1,\".txt\":70,\".yml\":1},\"requestId\":\"dc3cad34-9ece-49d6-b38e-28d8439f29e1\"},\"created\":\"2024-04-17T01:26:13.000Z\",\"event\":\"org.project.files.edit\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\"}" - }, - { - "message": "{\"content\":{\"action\":\"Retrieve files\",\"requestId\":\"dc3cad34-9ece-49d6-b38e-28d8439f29e1\"},\"created\":\"2024-04-17T01:26:13.000Z\",\"event\":\"org.project.files.access\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\"}" - }, - { - "message": "{\"content\":{\"action\":\"Returned from analysis\"},\"created\":\"2024-04-17T01:26:19.025Z\",\"event\":\"org.project.issue.create\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\"}" - }, - { - "message": "{\"content\":{\"issues\":7},\"created\":\"2024-04-17T01:26:19.192Z\",\"event\":\"org.project.issue.edit\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"project_id\":\"f6f87c14-594e-4335-b873-d3473054834d\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" - }, - { - "message": "{\"content\":{\"snapshotId\":\"7c4c1751-2f5e-4d79-96e4-5a200aa6d802\"},\"created\":\"2024-04-17T01:26:19.268Z\",\"event\":\"org.project.edit\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"project_id\":\"f6f87c14-594e-4335-b873-d3473054834d\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" - }, - { - "message": "{\"content\":{\"AboveSizeLimit\":{\".js\":1},\"action\":\"Modify files - exclude\",\"excluded\":{\"\":1,\".yml\":1},\"notSupported\":{\"\":163,\".0\":10,\".00\":1,\".04\":1,\".1\":1,\".1-faulty\":1,\".14\":2,\".17\":1,\".18-Debian\":1,\".2\":1,\".20\":1,\".23-CentOS6\":1,\".asciidoc\":1214,\".bash\":1,\".bat\":14,\".cert\":1,\".cfg\":1,\".cnf\":1,\".conf\":17,\".crt\":12,\".csr\":4,\".csv\":5,\".current\":2,\".dat\":88,\".debug\":1,\".disabled\":141,\".dll\":1,\".dockerignore\":2,\".editorconfig\":2,\".env\":1,\".evtx\":115,\".exe\":1,\".expected\":54,\".fbs\":1,\".gitattributes\":1,\".gitignore\":30,\".go-version\":1,\".gob\":1,\".groovy\":8,\".gz\":4,\".hcl\":2,\".ini\":2,\".j2\":14,\".jewel\":1,\".jks\":2,\".journal\":1,\".jpg\":10,\".json\":3500,\".key\":16,\".log\":590,\".md\":116,\".mk\":2,\".mmdb\":3,\".mod\":1,\".nautilus\":1,\".ndjson\":4,\".orig\":1,\".pcap\":118,\".pem\":8,\".pic\":6,\".placeholder\":3,\".plain\":51,\".png\":208,\".properties\":2,\".ps1\":1,\".pylintrc\":1,\".rl\":15,\".sh\":62,\".spec\":1,\".sql\":1,\".srl\":2,\".sum\":2,\".svg\":8,\".template\":2,\".tf\":21,\".thrift\":3,\".tmpl\":117,\".toml\":1,\".tpl\":1,\".txt\":32,\".xsl\":1,\".yaml\":34,\".yml\":1844,\".zip\":3},\"requestId\":\"dc3cad34-9ece-49d6-b38e-28d8439f29e1\"},\"created\":\"2024-04-17T01:26:21.000Z\",\"event\":\"org.project.files.edit\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\"}" - }, - { - "message": "{\"content\":{\"origin\":\"github\",\"target\":{\"branch\":\"dev\",\"id\":477916110,\"name\":\"mito\",\"owner\":\"elastic\"},\"targetFile\":\"go.mod\",\"type\":\"gomodules\"},\"created\":\"2024-04-17T01:26:29.493Z\",\"event\":\"org.project.monitor\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"project_id\":\"ad562805-e976-4eed-85b6-740c7664d607\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" - }, - { - "message": "{\"content\":{\"sourceOrgId\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\"},\"created\":\"2024-04-17T01:26:49.288Z\",\"event\":\"org.project.add\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"project_id\":\"2d92916c-792a-46b0-aa23-c94d7481478b\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" - }, - { - "message": "{\"content\":{\"issues\":7},\"created\":\"2024-04-17T01:35:51.787Z\",\"event\":\"org.project.issue.access\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"project_id\":\"f6f87c14-594e-4335-b873-d3473054834d\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" - }, - { - "message": "{\"content\":{},\"created\":\"2024-04-17T01:35:52.948Z\",\"event\":\"org.project.file.access\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"project_id\":\"f6f87c14-594e-4335-b873-d3473054834d\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" - } - ] + "events": [ + { + "message": "{\"content\":{\"after\":{\"name\":\"elastic-integration\"},\"before\":{\"name\":\"admin.user\"}},\"created\":\"2024-04-15T19:47:21.565Z\",\"event\":\"org.edit\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"user_id\":\"b4b324c4-a55c-4cd6-82b8-f96e3b3b8d85\"}" + }, + { + "message": "{\"content\":{\"email\":\"other.user@company.com\",\"role\":\"0afa84b4-9d99-4c2b-94c2-65da22270836\"},\"created\":\"2024-04-15T19:49:01.920Z\",\"event\":\"org.user.invite\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"user_id\":\"b4b324c4-a55c-4cd6-82b8-f96e3b3b8d85\"}" + }, + { + "message": "{\"content\":{\"openInvitePublicId\":\"5cc83241-4d28-401e-80e7-3a421cee2c03\",\"url\":\"https://app.snyk.io/invite/link/accept?invite=5cc83241-4d28-401e-80e7-3a421cee2c03\\u0026utm_source=link_invite\\u0026utm_medium=referral\\u0026utm_campaign=product-link-invite\\u0026from=link_invite\"},\"created\":\"2024-04-16T09:46:29.448Z\",\"event\":\"org.user.invite_link.create\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"user_id\":\"b4b324c4-a55c-4cd6-82b8-f96e3b3b8d85\"}" + }, + { + "message": "{\"content\":{\"role\":\"ADMIN\",\"rolePublicId\":\"0afa84b4-9d99-4c2b-94c2-65da22270836\",\"userPublicId\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"},\"created\":\"2024-04-16T21:54:33.257Z\",\"event\":\"org.user.add\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" + }, + { + "message": "{\"content\":{\"email\":\"other.user-alias@company.com\",\"invitingUserId\":1881478,\"rolePublicId\":\"0afa84b4-9d99-4c2b-94c2-65da22270836\"},\"created\":\"2024-04-16T21:54:33.257Z\",\"event\":\"org.user.invite_link.accept\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" + }, + { + "message": "{\"content\":{\"origin\":\"cli\"},\"created\":\"2024-04-17T01:08:08.228Z\",\"event\":\"org.project.test\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" + }, + { + "message": "{\"content\":{\"after\":{\"autoDepUpgradeEnabled\":true,\"autoRemediationPrs\":{\"backlogPrStrategy\":\"vuln\",\"container\":{\"enabled\":true},\"freshPrsEnabled\":true,\"usePatchRemediation\":true},\"pullRequestTestEnabled\":true,\"reachableVulns\":{}},\"before\":{\"autoDepUpgradeEnabled\":true,\"autoRemediationPrs\":{\"backlogPrStrategy\":\"vuln\",\"container\":{},\"freshPrsEnabled\":true,\"usePatchRemediation\":true},\"pullRequestTestEnabled\":true,\"reachableVulns\":{}},\"flow\":\"on-boarding\",\"sourceType\":\"github\"},\"created\":\"2024-04-17T01:24:49.748Z\",\"event\":\"org.integration.settings.edit\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" + }, + { + "message": "{\"content\":{\"after\":{\"sastSettings\":{\"sastEnabled\":true}},\"before\":{\"sastSettings\":{}},\"interface\":\"ui\"},\"created\":\"2024-04-17T01:24:49.837Z\",\"event\":\"org.sast_settings.edit\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" + }, + { + "message": "{\"content\":{\"targetId\":\"693b1550-43b1-4108-a55a-37e5cabe7355\"},\"created\":\"2024-04-17T01:26:10.024Z\",\"event\":\"org.target.create\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" + }, + { + "message": "{\"content\":{\"action\":\"Cloned repo: https://github.com/elastic/mito.git commit hash: 5e8963319b4a55b32f6d9db0a19f0ddd70ae8c5d\",\"requestId\":\"dc3cad34-9ece-49d6-b38e-28d8439f29e1\"},\"created\":\"2024-04-17T01:26:13.000Z\",\"event\":\"org.project.files.create\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\"}" + }, + { + "message": "{\"content\":{\"AboveSizeLimit\":{},\"action\":\"Modify files - exclude\",\"excluded\":{},\"notSupported\":{\"\":1,\".cel\":1,\".json\":1,\".md\":1,\".mod\":1,\".sum\":1,\".txt\":70,\".yml\":1},\"requestId\":\"dc3cad34-9ece-49d6-b38e-28d8439f29e1\"},\"created\":\"2024-04-17T01:26:13.000Z\",\"event\":\"org.project.files.edit\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\"}" + }, + { + "message": "{\"content\":{\"action\":\"Retrieve files\",\"requestId\":\"dc3cad34-9ece-49d6-b38e-28d8439f29e1\"},\"created\":\"2024-04-17T01:26:13.000Z\",\"event\":\"org.project.files.access\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\"}" + }, + { + "message": "{\"content\":{\"action\":\"Returned from analysis\"},\"created\":\"2024-04-17T01:26:19.025Z\",\"event\":\"org.project.issue.create\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\"}" + }, + { + "message": "{\"content\":{\"issues\":7},\"created\":\"2024-04-17T01:26:19.192Z\",\"event\":\"org.project.issue.edit\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"project_id\":\"f6f87c14-594e-4335-b873-d3473054834d\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" + }, + { + "message": "{\"content\":{\"snapshotId\":\"7c4c1751-2f5e-4d79-96e4-5a200aa6d802\"},\"created\":\"2024-04-17T01:26:19.268Z\",\"event\":\"org.project.edit\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"project_id\":\"f6f87c14-594e-4335-b873-d3473054834d\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" + }, + { + "message": "{\"content\":{\"AboveSizeLimit\":{\".js\":1},\"action\":\"Modify files - exclude\",\"excluded\":{\"\":1,\".yml\":1},\"notSupported\":{\"\":163,\".0\":10,\".00\":1,\".04\":1,\".1\":1,\".1-faulty\":1,\".14\":2,\".17\":1,\".18-Debian\":1,\".2\":1,\".20\":1,\".23-CentOS6\":1,\".asciidoc\":1214,\".bash\":1,\".bat\":14,\".cert\":1,\".cfg\":1,\".cnf\":1,\".conf\":17,\".crt\":12,\".csr\":4,\".csv\":5,\".current\":2,\".dat\":88,\".debug\":1,\".disabled\":141,\".dll\":1,\".dockerignore\":2,\".editorconfig\":2,\".env\":1,\".evtx\":115,\".exe\":1,\".expected\":54,\".fbs\":1,\".gitattributes\":1,\".gitignore\":30,\".go-version\":1,\".gob\":1,\".groovy\":8,\".gz\":4,\".hcl\":2,\".ini\":2,\".j2\":14,\".jewel\":1,\".jks\":2,\".journal\":1,\".jpg\":10,\".json\":3500,\".key\":16,\".log\":590,\".md\":116,\".mk\":2,\".mmdb\":3,\".mod\":1,\".nautilus\":1,\".ndjson\":4,\".orig\":1,\".pcap\":118,\".pem\":8,\".pic\":6,\".placeholder\":3,\".plain\":51,\".png\":208,\".properties\":2,\".ps1\":1,\".pylintrc\":1,\".rl\":15,\".sh\":62,\".spec\":1,\".sql\":1,\".srl\":2,\".sum\":2,\".svg\":8,\".template\":2,\".tf\":21,\".thrift\":3,\".tmpl\":117,\".toml\":1,\".tpl\":1,\".txt\":32,\".xsl\":1,\".yaml\":34,\".yml\":1844,\".zip\":3},\"requestId\":\"dc3cad34-9ece-49d6-b38e-28d8439f29e1\"},\"created\":\"2024-04-17T01:26:21.000Z\",\"event\":\"org.project.files.edit\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\"}" + }, + { + "message": "{\"content\":{\"origin\":\"github\",\"target\":{\"branch\":\"dev\",\"id\":477916110,\"name\":\"mito\",\"owner\":\"elastic\"},\"targetFile\":\"go.mod\",\"type\":\"gomodules\"},\"created\":\"2024-04-17T01:26:29.493Z\",\"event\":\"org.project.monitor\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"project_id\":\"ad562805-e976-4eed-85b6-740c7664d607\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" + }, + { + "message": "{\"content\":{\"sourceOrgId\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\"},\"created\":\"2024-04-17T01:26:49.288Z\",\"event\":\"org.project.add\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"project_id\":\"2d92916c-792a-46b0-aa23-c94d7481478b\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" + }, + { + "message": "{\"content\":{\"issues\":7},\"created\":\"2024-04-17T01:35:51.787Z\",\"event\":\"org.project.issue.access\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"project_id\":\"f6f87c14-594e-4335-b873-d3473054834d\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" + }, + { + "message": "{\"content\":{},\"created\":\"2024-04-17T01:35:52.948Z\",\"event\":\"org.project.file.access\",\"org_id\":\"86054e75-398a-4a1a-9dd8-72d026d8c237\",\"project_id\":\"f6f87c14-594e-4335-b873-d3473054834d\",\"user_id\":\"14b442f2-02f8-47d4-ba51-1749d3771e2c\"}" + } + ] } diff --git a/packages/snyk/data_stream/audit_logs/_dev/test/pipeline/test-snyk-audit.json-expected.json b/packages/snyk/data_stream/audit_logs/_dev/test/pipeline/test-snyk-audit.json-expected.json index 1208451483b..b2236700d75 100644 --- a/packages/snyk/data_stream/audit_logs/_dev/test/pipeline/test-snyk-audit.json-expected.json +++ b/packages/snyk/data_stream/audit_logs/_dev/test/pipeline/test-snyk-audit.json-expected.json @@ -3,7 +3,7 @@ { "@timestamp": "2024-04-15T19:47:21.565Z", "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "action": "org.edit", @@ -44,7 +44,7 @@ { "@timestamp": "2024-04-15T19:49:01.920Z", "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "action": "org.user.invite", @@ -85,7 +85,7 @@ { "@timestamp": "2024-04-16T09:46:29.448Z", "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "action": "org.user.invite_link.create", @@ -133,7 +133,7 @@ { "@timestamp": "2024-04-16T21:54:33.257Z", "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "action": "org.user.add", @@ -175,7 +175,7 @@ { "@timestamp": "2024-04-16T21:54:33.257Z", "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "action": "org.user.invite_link.accept", @@ -217,7 +217,7 @@ { "@timestamp": "2024-04-17T01:08:08.228Z", "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "action": "org.project.test", @@ -253,7 +253,7 @@ { "@timestamp": "2024-04-17T01:24:49.748Z", "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "action": "org.integration.settings.edit", @@ -314,7 +314,7 @@ { "@timestamp": "2024-04-17T01:24:49.837Z", "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "action": "org.sast_settings.edit", @@ -355,7 +355,7 @@ { "@timestamp": "2024-04-17T01:26:10.024Z", "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "action": "org.target.create", @@ -391,7 +391,7 @@ { "@timestamp": "2024-04-17T01:26:13.000Z", "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "action": "org.project.files.create", @@ -422,7 +422,7 @@ { "@timestamp": "2024-04-17T01:26:13.000Z", "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "action": "org.project.files.edit", @@ -463,7 +463,7 @@ { "@timestamp": "2024-04-17T01:26:13.000Z", "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "action": "org.project.files.access", @@ -494,7 +494,7 @@ { "@timestamp": "2024-04-17T01:26:19.025Z", "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "action": "org.project.issue.create", @@ -521,7 +521,7 @@ { "@timestamp": "2024-04-17T01:26:19.192Z", "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "action": "org.project.issue.edit", @@ -558,7 +558,7 @@ { "@timestamp": "2024-04-17T01:26:19.268Z", "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "action": "org.project.edit", @@ -595,7 +595,7 @@ { "@timestamp": "2024-04-17T01:26:21.000Z", "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "action": "org.project.files.edit", @@ -719,7 +719,7 @@ { "@timestamp": "2024-04-17T01:26:29.493Z", "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "action": "org.project.monitor", @@ -764,7 +764,7 @@ { "@timestamp": "2024-04-17T01:26:49.288Z", "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "action": "org.project.add", @@ -801,7 +801,7 @@ { "@timestamp": "2024-04-17T01:35:51.787Z", "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "action": "org.project.issue.access", @@ -838,7 +838,7 @@ { "@timestamp": "2024-04-17T01:35:52.948Z", "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "event": { "action": "org.project.file.access", diff --git a/packages/snyk/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml b/packages/snyk/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml index cb57c43fd31..9923057381e 100644 --- a/packages/snyk/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/snyk/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml @@ -3,7 +3,7 @@ description: Pipeline for Snyk Audit logs processors: - set: field: ecs.version - value: 8.12.0 + value: 8.11.0 - rename: field: message target_field: event.original diff --git a/packages/snyk/data_stream/audit_logs/fields/agent.yml b/packages/snyk/data_stream/audit_logs/fields/agent.yml index 4d9a6f7b362..bc42d0a853b 100644 --- a/packages/snyk/data_stream/audit_logs/fields/agent.yml +++ b/packages/snyk/data_stream/audit_logs/fields/agent.yml @@ -1,100 +1,9 @@ - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/snyk/data_stream/audit_logs/fields/beats.yml b/packages/snyk/data_stream/audit_logs/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/snyk/data_stream/audit_logs/fields/beats.yml +++ b/packages/snyk/data_stream/audit_logs/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/snyk/data_stream/audit_logs/fields/ecs.yml b/packages/snyk/data_stream/audit_logs/fields/ecs.yml deleted file mode 100644 index 4e829eae856..00000000000 --- a/packages/snyk/data_stream/audit_logs/fields/ecs.yml +++ /dev/null @@ -1,28 +0,0 @@ -- name: event.created - external: ecs -- name: event.original - external: ecs -- name: tags - external: ecs -- name: message - external: ecs -- name: ecs.version - external: ecs -- name: organization.id - external: ecs -- name: user.group.id - external: ecs -- name: user.id - external: ecs -- name: related.user - external: ecs -- name: url.domain - external: ecs -- name: url.original - external: ecs -- name: url.path - external: ecs -- name: url.query - external: ecs -- name: url.scheme - external: ecs diff --git a/packages/snyk/data_stream/audit_logs/sample_event.json b/packages/snyk/data_stream/audit_logs/sample_event.json index 841ca837f73..50d4cec4937 100644 --- a/packages/snyk/data_stream/audit_logs/sample_event.json +++ b/packages/snyk/data_stream/audit_logs/sample_event.json @@ -13,7 +13,7 @@ "type": "logs" }, "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "elastic_agent": { "id": "24936262-0cda-4934-aea3-82bed4844c98", diff --git a/packages/snyk/data_stream/issues/fields/agent.yml b/packages/snyk/data_stream/issues/fields/agent.yml index 4d9a6f7b362..bc42d0a853b 100644 --- a/packages/snyk/data_stream/issues/fields/agent.yml +++ b/packages/snyk/data_stream/issues/fields/agent.yml @@ -1,100 +1,9 @@ - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/snyk/data_stream/issues/fields/beats.yml b/packages/snyk/data_stream/issues/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/snyk/data_stream/issues/fields/beats.yml +++ b/packages/snyk/data_stream/issues/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/snyk/data_stream/issues/fields/ecs.yml b/packages/snyk/data_stream/issues/fields/ecs.yml deleted file mode 100644 index 74148055477..00000000000 --- a/packages/snyk/data_stream/issues/fields/ecs.yml +++ /dev/null @@ -1,34 +0,0 @@ -- name: event.created - external: ecs -- name: event.original - external: ecs -- name: tags - external: ecs -- name: message - external: ecs -- name: ecs.version - external: ecs -- name: user.group.id - external: ecs -- name: user.id - external: ecs -- name: organization.id - external: ecs -- name: vulnerability.category - external: ecs -- name: vulnerability.classification - external: ecs -- name: vulnerability.enumeration - external: ecs -- name: vulnerability.id - external: ecs -- name: vulnerability.reference - external: ecs -- name: vulnerability.scanner.vendor - external: ecs -- name: vulnerability.score.base - external: ecs -- name: vulnerability.score.version - external: ecs -- name: vulnerability.severity - external: ecs diff --git a/packages/snyk/data_stream/vulnerabilities/fields/agent.yml b/packages/snyk/data_stream/vulnerabilities/fields/agent.yml index 4d9a6f7b362..bc42d0a853b 100644 --- a/packages/snyk/data_stream/vulnerabilities/fields/agent.yml +++ b/packages/snyk/data_stream/vulnerabilities/fields/agent.yml @@ -1,100 +1,9 @@ - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/snyk/data_stream/vulnerabilities/fields/beats.yml b/packages/snyk/data_stream/vulnerabilities/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/snyk/data_stream/vulnerabilities/fields/beats.yml +++ b/packages/snyk/data_stream/vulnerabilities/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/snyk/data_stream/vulnerabilities/fields/ecs.yml b/packages/snyk/data_stream/vulnerabilities/fields/ecs.yml deleted file mode 100644 index 0e1e5e77d09..00000000000 --- a/packages/snyk/data_stream/vulnerabilities/fields/ecs.yml +++ /dev/null @@ -1,32 +0,0 @@ -- name: event.created - external: ecs -- name: event.original - external: ecs -- name: tags - external: ecs -- name: message - external: ecs -- name: ecs.version - external: ecs -- name: user.group.id - external: ecs -- name: user.id - external: ecs -- name: vulnerability.category - external: ecs -- name: vulnerability.classification - external: ecs -- name: vulnerability.enumeration - external: ecs -- name: vulnerability.id - external: ecs -- name: vulnerability.reference - external: ecs -- name: vulnerability.scanner.vendor - external: ecs -- name: vulnerability.score.base - external: ecs -- name: vulnerability.score.version - external: ecs -- name: vulnerability.severity - external: ecs diff --git a/packages/snyk/docs/README.md b/packages/snyk/docs/README.md index 47259f69798..a9e01311792 100644 --- a/packages/snyk/docs/README.md +++ b/packages/snyk/docs/README.md @@ -44,7 +44,7 @@ An example event for `audit` looks as following: "type": "logs" }, "ecs": { - "version": "8.12.0" + "version": "8.11.0" }, "elastic_agent": { "id": "24936262-0cda-4934-aea3-82bed4844c98", @@ -92,50 +92,20 @@ An example event for `audit` looks as following: | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| organization.id | Unique identifier for the organization. | keyword | -| related.user | All the user names or other user identifiers seen on the event. | keyword | | snyk.audit_logs.content | Overview of the content that was changed, both old and new values. | flattened | | snyk.audit_logs.org_id | ID of the related Organization related to the event. | keyword | | snyk.audit_logs.project_id | ID of the project related to the event. | keyword | | snyk.audit_logs.user_id | ID of the user related to the event. | keyword | | snyk.projects | Array with all related projects objects. | flattened | | snyk.related.projects | Array of all the related project ID's. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| user.group.id | Unique identifier for the group on the system/platform. | keyword | -| user.id | Unique identifier of the user. | keyword | ## Issues @@ -308,34 +278,14 @@ An example event for `issues` looks as following: | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| organization.id | Unique identifier for the organization. | keyword | | snyk.issues.attributes.classes.id | | keyword | | snyk.issues.attributes.classes.source | | keyword | | snyk.issues.attributes.classes.type | | keyword | @@ -377,18 +327,6 @@ An example event for `issues` looks as following: | snyk.issues.relationships.scan_item.links.related | | keyword | | snyk.projects | Array with all related projects objects. | flattened | | snyk.related.projects | Array of all the related project ID's. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| user.group.id | Unique identifier for the group on the system/platform. | keyword | -| user.id | Unique identifier of the user. | keyword | -| vulnerability.category | The type of system or architecture that the vulnerability affects. These may be platform-specific (for example, Debian or SUSE) or general (for example, Database or Firewall). For example (https://qualysguard.qualys.com/qwebhelp/fo_portal/knowledgebase/vulnerability_categories.htm[Qualys vulnerability categories]) This field must be an array. | keyword | -| vulnerability.classification | The classification of the vulnerability scoring system. For example (https://www.first.org/cvss/) | keyword | -| vulnerability.enumeration | The type of identifier used for this vulnerability. For example (https://cve.mitre.org/about/) | keyword | -| vulnerability.id | The identification (ID) is the number portion of a vulnerability entry. It includes a unique identification number for the vulnerability. For example (https://cve.mitre.org/about/faqs.html#what_is_cve_id)[Common Vulnerabilities and Exposure CVE ID] | keyword | -| vulnerability.reference | A resource that provides additional information, context, and mitigations for the identified vulnerability. | keyword | -| vulnerability.scanner.vendor | The name of the vulnerability scanner vendor. | keyword | -| vulnerability.score.base | Scores can range from 0.0 to 10.0, with 10.0 being the most severe. Base scores cover an assessment for exploitability metrics (attack vector, complexity, privileges, and user interaction), impact metrics (confidentiality, integrity, and availability), and scope. For example (https://www.first.org/cvss/specification-document) | float | -| vulnerability.score.version | The National Vulnerability Database (NVD) provides qualitative severity rankings of "Low", "Medium", and "High" for CVSS v2.0 base score ranges in addition to the severity ratings for CVSS v3.0 as they are defined in the CVSS v3.0 specification. CVSS is owned and managed by FIRST.Org, Inc. (FIRST), a US-based non-profit organization, whose mission is to help computer security incident response teams across the world. For example (https://nvd.nist.gov/vuln-metrics/cvss) | keyword | -| vulnerability.severity | The severity of the vulnerability can help with metrics and internal prioritization regarding remediation. For example (https://nvd.nist.gov/vuln-metrics/cvss) | keyword | ## Audit (Legacy) @@ -460,41 +398,19 @@ An example event for `audit` looks as following: | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | snyk.audit.content | Overview of the content that was changed, both old and new values. | flattened | | snyk.audit.org_id | ID of the related Organization related to the event. | keyword | | snyk.audit.project_id | ID of the project related to the event. | keyword | | snyk.projects | Array with all related projects objects. | flattened | | snyk.related.projects | Array of all the related project ID's. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| user.group.id | Unique identifier for the group on the system/platform. | keyword | -| user.id | Unique identifier of the user. | keyword | ## Vulnerabilities (Legacy) @@ -653,33 +569,14 @@ An example event for `vulnerabilities` looks as following: | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | snyk.projects | Array with all related projects objects. | flattened | | snyk.related.projects | Array of all the related project ID's. | keyword | | snyk.vulnerabilities.credit | Reference to the person that original found the vulnerability. | keyword | @@ -710,17 +607,5 @@ An example event for `vulnerabilities` looks as following: | snyk.vulnerabilities.type | The issue type. Can be either "license" or "vulnerability". | keyword | | snyk.vulnerabilities.unique_severities_list | A list of related unique severities. | keyword | | snyk.vulnerabilities.version | The package version this issue is applicable to. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| user.group.id | Unique identifier for the group on the system/platform. | keyword | -| user.id | Unique identifier of the user. | keyword | -| vulnerability.category | The type of system or architecture that the vulnerability affects. These may be platform-specific (for example, Debian or SUSE) or general (for example, Database or Firewall). For example (https://qualysguard.qualys.com/qwebhelp/fo_portal/knowledgebase/vulnerability_categories.htm[Qualys vulnerability categories]) This field must be an array. | keyword | -| vulnerability.classification | The classification of the vulnerability scoring system. For example (https://www.first.org/cvss/) | keyword | -| vulnerability.enumeration | The type of identifier used for this vulnerability. For example (https://cve.mitre.org/about/) | keyword | -| vulnerability.id | The identification (ID) is the number portion of a vulnerability entry. It includes a unique identification number for the vulnerability. For example (https://cve.mitre.org/about/faqs.html#what_is_cve_id)[Common Vulnerabilities and Exposure CVE ID] | keyword | -| vulnerability.reference | A resource that provides additional information, context, and mitigations for the identified vulnerability. | keyword | -| vulnerability.scanner.vendor | The name of the vulnerability scanner vendor. | keyword | -| vulnerability.score.base | Scores can range from 0.0 to 10.0, with 10.0 being the most severe. Base scores cover an assessment for exploitability metrics (attack vector, complexity, privileges, and user interaction), impact metrics (confidentiality, integrity, and availability), and scope. For example (https://www.first.org/cvss/specification-document) | float | -| vulnerability.score.version | The National Vulnerability Database (NVD) provides qualitative severity rankings of "Low", "Medium", and "High" for CVSS v2.0 base score ranges in addition to the severity ratings for CVSS v3.0 as they are defined in the CVSS v3.0 specification. CVSS is owned and managed by FIRST.Org, Inc. (FIRST), a US-based non-profit organization, whose mission is to help computer security incident response teams across the world. For example (https://nvd.nist.gov/vuln-metrics/cvss) | keyword | -| vulnerability.severity | The severity of the vulnerability can help with metrics and internal prioritization regarding remediation. For example (https://nvd.nist.gov/vuln-metrics/cvss) | keyword | diff --git a/packages/snyk/manifest.yml b/packages/snyk/manifest.yml index 1105a9de36e..3ae3b795265 100644 --- a/packages/snyk/manifest.yml +++ b/packages/snyk/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: snyk title: "Snyk" -version: "1.22.1" +version: "1.23.0" description: Collect logs from Snyk with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - cloudsecurity_cdr conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" icons: - src: /img/snyk-logo.svg title: Snyk logo From 55519bb8c028eda777931c42fbe686b4815f329f Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:55 +0930 Subject: [PATCH 085/121] [sophos_central] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/sophos_central --- packages/sophos_central/_dev/build/build.yml | 1 - packages/sophos_central/changelog.yml | 5 +++++ packages/sophos_central/data_stream/alert/fields/beats.yml | 3 --- packages/sophos_central/data_stream/event/fields/beats.yml | 3 --- packages/sophos_central/docs/README.md | 2 -- packages/sophos_central/manifest.yml | 4 ++-- 6 files changed, 7 insertions(+), 11 deletions(-) diff --git a/packages/sophos_central/_dev/build/build.yml b/packages/sophos_central/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/sophos_central/_dev/build/build.yml +++ b/packages/sophos_central/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/sophos_central/changelog.yml b/packages/sophos_central/changelog.yml index 61e07f75170..be4a3c60012 100644 --- a/packages/sophos_central/changelog.yml +++ b/packages/sophos_central/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.15.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.14.0" changes: - description: Set sensitive values as secret and fix incorrect mapping. diff --git a/packages/sophos_central/data_stream/alert/fields/beats.yml b/packages/sophos_central/data_stream/alert/fields/beats.yml index 80cbae91cae..cc9fcebf29b 100644 --- a/packages/sophos_central/data_stream/alert/fields/beats.yml +++ b/packages/sophos_central/data_stream/alert/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/sophos_central/data_stream/event/fields/beats.yml b/packages/sophos_central/data_stream/event/fields/beats.yml index 80cbae91cae..cc9fcebf29b 100644 --- a/packages/sophos_central/data_stream/event/fields/beats.yml +++ b/packages/sophos_central/data_stream/event/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/sophos_central/docs/README.md b/packages/sophos_central/docs/README.md index 7df48147f80..6e5d2c47554 100644 --- a/packages/sophos_central/docs/README.md +++ b/packages/sophos_central/docs/README.md @@ -353,7 +353,6 @@ An example event for `alert` looks as following: | sophos_central.alert.threat.value | Name of the threat (as identified by threat_id). | keyword | | sophos_central.alert.type | Event type. | keyword | | sophos_central.alert.when | The date at which the alert was created. | date | -| tags | User defined tags. | keyword | ### Events @@ -570,4 +569,3 @@ An example event for `event` looks as following: | sophos_central.event.type | The type of this record. | keyword | | sophos_central.event.user_id | The identifier of the user for which record is created. | keyword | | sophos_central.event.when | The date at which the event was created. | date | -| tags | User defined tags. | keyword | diff --git a/packages/sophos_central/manifest.yml b/packages/sophos_central/manifest.yml index 2b47e94fc0c..aef6d3a8b9e 100644 --- a/packages/sophos_central/manifest.yml +++ b/packages/sophos_central/manifest.yml @@ -1,14 +1,14 @@ format_version: "3.0.2" name: sophos_central title: Sophos Central -version: "1.14.0" +version: "1.15.0" description: This Elastic integration collects logs from Sophos Central with Elastic Agent. type: integration categories: - security conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: subscription: "basic" screenshots: From 80d9c8ebfd38a5133dc50fddb60e1fb75f5410ce Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:56 +0930 Subject: [PATCH 086/121] [symantec_edr_cloud] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/symantec_edr_cloud --- packages/symantec_edr_cloud/_dev/build/build.yml | 1 - packages/symantec_edr_cloud/changelog.yml | 5 +++++ .../symantec_edr_cloud/data_stream/incident/fields/beats.yml | 3 --- packages/symantec_edr_cloud/docs/README.md | 1 - packages/symantec_edr_cloud/manifest.yml | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/symantec_edr_cloud/_dev/build/build.yml b/packages/symantec_edr_cloud/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/symantec_edr_cloud/_dev/build/build.yml +++ b/packages/symantec_edr_cloud/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/symantec_edr_cloud/changelog.yml b/packages/symantec_edr_cloud/changelog.yml index ad4bffb350a..50f76ab8f41 100644 --- a/packages/symantec_edr_cloud/changelog.yml +++ b/packages/symantec_edr_cloud/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.2.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.1.0" changes: - description: Set sensitive values as secret. diff --git a/packages/symantec_edr_cloud/data_stream/incident/fields/beats.yml b/packages/symantec_edr_cloud/data_stream/incident/fields/beats.yml index 80cbae91cae..cc9fcebf29b 100644 --- a/packages/symantec_edr_cloud/data_stream/incident/fields/beats.yml +++ b/packages/symantec_edr_cloud/data_stream/incident/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/symantec_edr_cloud/docs/README.md b/packages/symantec_edr_cloud/docs/README.md index e3c2ca386f0..8126cac0e25 100644 --- a/packages/symantec_edr_cloud/docs/README.md +++ b/packages/symantec_edr_cloud/docs/README.md @@ -225,5 +225,4 @@ An example event for `incident` looks as following: | symantec_edr_cloud.incident.type | Event type. | keyword | | symantec_edr_cloud.incident.type_id | | keyword | | symantec_edr_cloud.incident.version | API version in the form major.minor. | keyword | -| tags | User defined tags. | keyword | diff --git a/packages/symantec_edr_cloud/manifest.yml b/packages/symantec_edr_cloud/manifest.yml index 7c72366669f..3582b346960 100644 --- a/packages/symantec_edr_cloud/manifest.yml +++ b/packages/symantec_edr_cloud/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.2 name: symantec_edr_cloud title: Symantec EDR Cloud -version: "1.1.0" +version: "1.2.0" source: license: Elastic-2.0 description: Collect logs from Symantec EDR Cloud with Elastic Agent. @@ -10,7 +10,7 @@ categories: - security conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: subscription: basic screenshots: From c4a82026df2ab5c79b5963376d578e6137dfd318 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:26:58 +0930 Subject: [PATCH 087/121] [symantec_endpoint] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^7.16.0 || ^8.0.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/symantec_endpoint --- packages/symantec_endpoint/changelog.yml | 5 + .../pipeline/test-rfc3164.log-expected.json | 2 +- .../data_stream/log/fields/agent.yml | 164 +----------------- .../data_stream/log/fields/ecs.yml | 158 ----------------- packages/symantec_endpoint/docs/README.md | 113 ------------ packages/symantec_endpoint/manifest.yml | 4 +- 6 files changed, 9 insertions(+), 437 deletions(-) delete mode 100644 packages/symantec_endpoint/data_stream/log/fields/ecs.yml diff --git a/packages/symantec_endpoint/changelog.yml b/packages/symantec_endpoint/changelog.yml index 459db0dc3f5..d75e408aff7 100644 --- a/packages/symantec_endpoint/changelog.yml +++ b/packages/symantec_endpoint/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.16.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.15.0" changes: - description: Update manifest format version to v3.0.3. diff --git a/packages/symantec_endpoint/data_stream/log/_dev/test/pipeline/test-rfc3164.log-expected.json b/packages/symantec_endpoint/data_stream/log/_dev/test/pipeline/test-rfc3164.log-expected.json index 989812ecdcc..2ce19f8f144 100644 --- a/packages/symantec_endpoint/data_stream/log/_dev/test/pipeline/test-rfc3164.log-expected.json +++ b/packages/symantec_endpoint/data_stream/log/_dev/test/pipeline/test-rfc3164.log-expected.json @@ -1,7 +1,7 @@ { "expected": [ { - "@timestamp": "2023-10-04T10:51:33.000Z", + "@timestamp": "2024-10-04T10:51:33.000Z", "destination": { "address": "216.160.83.61", "as": { diff --git a/packages/symantec_endpoint/data_stream/log/fields/agent.yml b/packages/symantec_endpoint/data_stream/log/fields/agent.yml index 49dbf0d0e94..32012c4eba4 100644 --- a/packages/symantec_endpoint/data_stream/log/fields/agent.yml +++ b/packages/symantec_endpoint/data_stream/log/fields/agent.yml @@ -5,177 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - external: ecs - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/symantec_endpoint/data_stream/log/fields/ecs.yml b/packages/symantec_endpoint/data_stream/log/fields/ecs.yml deleted file mode 100644 index 20a9d61362b..00000000000 --- a/packages/symantec_endpoint/data_stream/log/fields/ecs.yml +++ /dev/null @@ -1,158 +0,0 @@ -- name: destination.address - external: ecs -- name: destination.as.number - external: ecs -- name: destination.as.organization.name - external: ecs -- name: destination.domain - external: ecs -- name: destination.geo.city_name - external: ecs -- name: destination.geo.continent_name - external: ecs -- name: destination.geo.country_iso_code - external: ecs -- name: destination.geo.country_name - external: ecs -- name: destination.geo.location - external: ecs -- name: destination.geo.name - external: ecs -- name: destination.geo.region_iso_code - external: ecs -- name: destination.geo.region_name - external: ecs -- name: destination.ip - external: ecs -- name: destination.mac - external: ecs -- name: destination.packets - external: ecs -- name: destination.port - external: ecs -- name: ecs.version - external: ecs -- name: error.message - external: ecs -- name: event.category - external: ecs -- name: event.ingested - external: ecs -- name: event.kind - external: ecs -- name: event.start - external: ecs -- name: event.type - external: ecs -- name: file.hash.sha1 - external: ecs -- name: file.name - external: ecs -- name: file.path - external: ecs -- name: file.pe.company - external: ecs -- name: file.pe.file_version - external: ecs -- name: file.pe.product - external: ecs -- name: file.size - external: ecs -- name: file.x509.issuer.common_name - external: ecs -- name: file.x509.not_before - external: ecs -- name: file.x509.serial_number - external: ecs -- name: log.file.path - external: ecs -- name: log.level - external: ecs -- name: log.syslog.appname - external: ecs -- name: log.syslog.hostname - external: ecs -- name: log.syslog.priority - external: ecs -- name: log.syslog.procid - external: ecs -- name: log.syslog.structured_data - external: ecs -- name: log.syslog.version - external: ecs -- name: message - external: ecs -- name: network.community_id - external: ecs -- name: network.direction - external: ecs -- name: network.transport - external: ecs -- name: network.type - external: ecs -- name: process.executable - external: ecs -- name: process.hash.md5 - external: ecs -- name: process.hash.sha256 - external: ecs -- name: process.name - external: ecs -- name: process.pid - external: ecs -- name: related.hash - external: ecs -- name: related.ip - external: ecs -- name: related.user - external: ecs -- name: rule.id - external: ecs -- name: rule.name - external: ecs -- name: source.address - external: ecs -- name: source.as.number - external: ecs -- name: source.as.organization.name - external: ecs -- name: source.domain - external: ecs -- name: source.geo.city_name - external: ecs -- name: source.geo.continent_name - external: ecs -- name: source.geo.country_iso_code - external: ecs -- name: source.geo.country_name - external: ecs -- name: source.geo.location - external: ecs -- name: source.geo.name - external: ecs -- name: source.geo.region_iso_code - external: ecs -- name: source.geo.region_name - external: ecs -- name: source.ip - external: ecs -- name: source.mac - external: ecs -- name: source.port - external: ecs -- name: tags - external: ecs -- name: url.domain - external: ecs -- name: url.original - external: ecs -- name: url.path - external: ecs -- name: url.scheme - external: ecs -- name: user.domain - external: ecs -- name: user.name - external: ecs -- name: user_agent.original - external: ecs diff --git a/packages/symantec_endpoint/docs/README.md b/packages/symantec_endpoint/docs/README.md index b06a6b186a8..a52191ad0c8 100644 --- a/packages/symantec_endpoint/docs/README.md +++ b/packages/symantec_endpoint/docs/README.md @@ -120,125 +120,23 @@ See vendor documentation: [External Logging settings and log event severity leve | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.mac | MAC address of the destination. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| destination.packets | Packets sent from the destination to the source. | long | -| destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | | event.dataset | Name of the dataset. | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Name of the module this data is coming from. | constant_keyword | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.hash.sha1 | SHA1 hash. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| file.pe.company | Internal company name of the file, provided at compile-time. | keyword | -| file.pe.file_version | Internal version of the file, provided at compile-time. | keyword | -| file.pe.product | Internal product name of the file, provided at compile-time. | keyword | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| file.x509.issuer.common_name | List of common name (CN) of issuing certificate authority. | keyword | -| file.x509.not_before | Time at which the certificate is first considered valid. | date | -| file.x509.serial_number | Unique serial number issued by the certificate authority. For consistency, if this value is alphanumeric, it should be formatted without colons and uppercase characters. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type. | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | -| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | | log.offset | Offset of the entry in the log file. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| log.syslog.appname | The device or application that originated the Syslog message, if available. | keyword | -| log.syslog.hostname | The hostname, FQDN, or IP of the machine that originally sent the Syslog message. This is sourced from the hostname field of the syslog header. Depending on the environment, this value may be different from the host that handled the event, especially if the host handling the events is acting as a collector. | keyword | -| log.syslog.priority | Syslog numeric priority of the event, if available. According to RFCs 5424 and 3164, the priority is 8 \* facility + severity. This number is therefore expected to contain a value between 0 and 191. | long | | log.syslog.process.name | Deprecated. Use the ECS log.syslog.appname field. | alias | | log.syslog.process.pid | Deprecated. Use the ECS log.syslog.procid field. | long | -| log.syslog.procid | The process name or ID that originated the Syslog message, if available. | keyword | -| log.syslog.structured_data | Structured data expressed in RFC 5424 messages, if available. These are key-value pairs formed from the structured data portion of the syslog message, as defined in RFC 5424 Section 6.3. | flattened | -| log.syslog.version | The version of the Syslog protocol specification. Only applicable for RFC 5424 messages. | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | | observer.product | The product name of the observer. | constant_keyword | | observer.type | The type of the observer the data is coming from. | constant_keyword | | observer.vendor | Vendor name of the observer. | constant_keyword | -| process.executable | Absolute path to the process executable. | keyword | -| process.executable.text | Multi-field of `process.executable`. | match_only_text | -| process.hash.md5 | MD5 hash. | keyword | -| process.hash.sha256 | SHA256 hash. | keyword | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.pid | Process id. | long | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.mac | MAC address of the source. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| source.port | Port of the source. | long | | symantec_endpoint.log.action | The action taken on the traffic, e.g. "Blocked". | keyword | | symantec_endpoint.log.actual_action | Actual action from risk logs and proactive detection (SONAR) logs. | keyword | | symantec_endpoint.log.admin | Name of the SEPM admin. | keyword | @@ -338,17 +236,6 @@ See vendor documentation: [External Logging settings and log event severity leve | symantec_endpoint.log.user2 | User when scan ended. | keyword | | symantec_endpoint.log.user_name | | keyword | | symantec_endpoint.log.web_domain | The web domain. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | An example event for `log` looks as following: diff --git a/packages/symantec_endpoint/manifest.yml b/packages/symantec_endpoint/manifest.yml index 0955664c4c3..b802cb2c24c 100644 --- a/packages/symantec_endpoint/manifest.yml +++ b/packages/symantec_endpoint/manifest.yml @@ -1,13 +1,13 @@ name: symantec_endpoint title: Symantec Endpoint Protection -version: "2.15.0" +version: "2.16.0" description: Collect logs from Symantec Endpoint Protection with Elastic Agent. type: integration format_version: "3.0.3" categories: ["security", "edr_xdr"] conditions: kibana: - version: "^7.16.0 || ^8.0.0" + version: "^8.13.0" icons: - src: /img/logo.svg title: Symantec From 733965e19d81c855eecfb59e38e81135c7a131f4 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:27:28 +0930 Subject: [PATCH 088/121] [symantec_endpoint_security] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/symantec_endpoint_security --- packages/symantec_endpoint_security/_dev/build/build.yml | 1 - packages/symantec_endpoint_security/changelog.yml | 5 +++++ .../data_stream/event/fields/beats.yml | 3 --- packages/symantec_endpoint_security/docs/README.md | 1 - packages/symantec_endpoint_security/manifest.yml | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/symantec_endpoint_security/_dev/build/build.yml b/packages/symantec_endpoint_security/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/symantec_endpoint_security/_dev/build/build.yml +++ b/packages/symantec_endpoint_security/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/symantec_endpoint_security/changelog.yml b/packages/symantec_endpoint_security/changelog.yml index 5d42a97fba0..de78a93cc4a 100644 --- a/packages/symantec_endpoint_security/changelog.yml +++ b/packages/symantec_endpoint_security/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.2.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "0.1.0" changes: - description: Initial release. diff --git a/packages/symantec_endpoint_security/data_stream/event/fields/beats.yml b/packages/symantec_endpoint_security/data_stream/event/fields/beats.yml index 083dcfe307e..fff1b3f1b6b 100644 --- a/packages/symantec_endpoint_security/data_stream/event/fields/beats.yml +++ b/packages/symantec_endpoint_security/data_stream/event/fields/beats.yml @@ -4,9 +4,6 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. - name: aws.s3 type: group fields: diff --git a/packages/symantec_endpoint_security/docs/README.md b/packages/symantec_endpoint_security/docs/README.md index 0500b26fac5..112168e98c7 100644 --- a/packages/symantec_endpoint_security/docs/README.md +++ b/packages/symantec_endpoint_security/docs/README.md @@ -3398,5 +3398,4 @@ An example event for `event` looks as following: | ses.verdict_id | The outcome of the Scan. | keyword | | ses.verdict_value | The outcome value of the Scan. | keyword | | ses.version | The event type version, in the form major.minor. | keyword | -| tags | User defined tags. | keyword | diff --git a/packages/symantec_endpoint_security/manifest.yml b/packages/symantec_endpoint_security/manifest.yml index 1f4e26c5bbe..01a9d075934 100644 --- a/packages/symantec_endpoint_security/manifest.yml +++ b/packages/symantec_endpoint_security/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.3 name: symantec_endpoint_security title: Symantec Endpoint Security -version: 0.1.0 +version: "0.2.0" description: Collect logs from Symantec Endpoint Security with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - edr_xdr conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: subscription: "basic" screenshots: From 56c427e251db09e9420dba344546663a186dcc4b Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:27:33 +0930 Subject: [PATCH 089/121] [tanium] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/tanium --- packages/tanium/_dev/build/build.yml | 1 - packages/tanium/changelog.yml | 5 +++++ packages/tanium/data_stream/action_history/fields/beats.yml | 3 --- packages/tanium/data_stream/client_status/fields/beats.yml | 3 --- packages/tanium/data_stream/discover/fields/beats.yml | 3 --- .../tanium/data_stream/endpoint_config/fields/beats.yml | 3 --- packages/tanium/data_stream/reporting/fields/beats.yml | 3 --- .../tanium/data_stream/threat_response/fields/beats.yml | 3 --- packages/tanium/docs/README.md | 6 ------ packages/tanium/manifest.yml | 4 ++-- 10 files changed, 7 insertions(+), 27 deletions(-) diff --git a/packages/tanium/_dev/build/build.yml b/packages/tanium/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/tanium/_dev/build/build.yml +++ b/packages/tanium/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/tanium/changelog.yml b/packages/tanium/changelog.yml index f515613a613..254d69cb362 100644 --- a/packages/tanium/changelog.yml +++ b/packages/tanium/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.10.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.9.1" changes: - description: Resolved ignore_malformed issues with fields. diff --git a/packages/tanium/data_stream/action_history/fields/beats.yml b/packages/tanium/data_stream/action_history/fields/beats.yml index 1214b97a0c7..28e147d9c90 100644 --- a/packages/tanium/data_stream/action_history/fields/beats.yml +++ b/packages/tanium/data_stream/action_history/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/tanium/data_stream/client_status/fields/beats.yml b/packages/tanium/data_stream/client_status/fields/beats.yml index 1214b97a0c7..28e147d9c90 100644 --- a/packages/tanium/data_stream/client_status/fields/beats.yml +++ b/packages/tanium/data_stream/client_status/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/tanium/data_stream/discover/fields/beats.yml b/packages/tanium/data_stream/discover/fields/beats.yml index 1214b97a0c7..28e147d9c90 100644 --- a/packages/tanium/data_stream/discover/fields/beats.yml +++ b/packages/tanium/data_stream/discover/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/tanium/data_stream/endpoint_config/fields/beats.yml b/packages/tanium/data_stream/endpoint_config/fields/beats.yml index 1214b97a0c7..28e147d9c90 100644 --- a/packages/tanium/data_stream/endpoint_config/fields/beats.yml +++ b/packages/tanium/data_stream/endpoint_config/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/tanium/data_stream/reporting/fields/beats.yml b/packages/tanium/data_stream/reporting/fields/beats.yml index 1214b97a0c7..28e147d9c90 100644 --- a/packages/tanium/data_stream/reporting/fields/beats.yml +++ b/packages/tanium/data_stream/reporting/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/tanium/data_stream/threat_response/fields/beats.yml b/packages/tanium/data_stream/threat_response/fields/beats.yml index 1214b97a0c7..28e147d9c90 100644 --- a/packages/tanium/data_stream/threat_response/fields/beats.yml +++ b/packages/tanium/data_stream/threat_response/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/tanium/docs/README.md b/packages/tanium/docs/README.md index a07beb31254..ae30bdf8645 100644 --- a/packages/tanium/docs/README.md +++ b/packages/tanium/docs/README.md @@ -165,7 +165,6 @@ An example event for `action_history` looks as following: | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | User defined tags. | keyword | | tanium.action_history.action.id | Action Id. | long | | tanium.action_history.action.name | Action Name. | keyword | | tanium.action_history.approver | Approver of the action. | keyword | @@ -270,7 +269,6 @@ An example event for `client_status` looks as following: | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | User defined tags. | keyword | | tanium.client_status.client_network_location | Network location of client. | ip | | tanium.client_status.computer_id | Computer ID of client. | keyword | | tanium.client_status.full_version | Full version of client. | version | @@ -374,7 +372,6 @@ An example event for `discover` looks as following: | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | User defined tags. | keyword | | tanium.discover.arp | Address Resolution Protocol. | double | | tanium.discover.aws_api | Aws Api version. | double | | tanium.discover.centralized_nmap | Centralized Nmap. | double | @@ -509,7 +506,6 @@ An example event for `endpoint_config` looks as following: | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | User defined tags. | keyword | | tanium.endpoint_config.action | Name of event's action. | keyword | | tanium.endpoint_config.item.data_category | Data category of the config item. | keyword | | tanium.endpoint_config.item.domain | Domain of the config item. | keyword | @@ -631,7 +627,6 @@ An example event for `reporting` looks as following: | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | User defined tags. | keyword | | tanium.reporting.computer_name | Name of the computer. | keyword | | tanium.reporting.count | Count of report on the computer system. | long | | tanium.reporting.is_virtual | Boolean flag mentions if computer is virtualise or not. | keyword | @@ -976,7 +971,6 @@ An example event for `threat_response` looks as following: | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | User defined tags. | keyword | | tanium.threat_response.action | Action for the threat response. | keyword | | tanium.threat_response.computer.ip | Computer ip of the threat response. | ip | | tanium.threat_response.computer.name | Computer name of the threat response. | keyword | diff --git a/packages/tanium/manifest.yml b/packages/tanium/manifest.yml index 4ef85147d95..466f341455c 100644 --- a/packages/tanium/manifest.yml +++ b/packages/tanium/manifest.yml @@ -1,14 +1,14 @@ format_version: "3.0.3" name: tanium title: Tanium -version: "1.9.1" +version: "1.10.0" description: This Elastic integration collects logs from Tanium with Elastic Agent. type: integration categories: - security conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: subscription: "basic" screenshots: From d95e665745c5d02682194f92a65aad3155ebe4ac Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:27:36 +0930 Subject: [PATCH 090/121] [tenable_io] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/tenable_io --- packages/tenable_io/changelog.yml | 5 + .../data_stream/asset/fields/agent.yml | 167 +----------------- .../data_stream/asset/fields/ecs.yml | 20 --- .../asset/fields/overridden-ecs.yml | 4 - .../data_stream/plugin/fields/agent.yml | 167 +----------------- .../data_stream/plugin/fields/ecs.yml | 22 --- .../plugin/fields/overridden-ecs.yml | 4 - .../data_stream/scan/fields/agent.yml | 167 +----------------- .../data_stream/scan/fields/ecs.yml | 18 -- .../scan/fields/overridden-ecs.yml | 4 - .../vulnerability/fields/agent.yml | 167 +----------------- .../data_stream/vulnerability/fields/ecs.yml | 40 ----- .../vulnerability/fields/overridden-ecs.yml | 4 - packages/tenable_io/docs/README.md | 158 ----------------- packages/tenable_io/manifest.yml | 4 +- 15 files changed, 11 insertions(+), 940 deletions(-) delete mode 100644 packages/tenable_io/data_stream/asset/fields/ecs.yml delete mode 100644 packages/tenable_io/data_stream/asset/fields/overridden-ecs.yml delete mode 100644 packages/tenable_io/data_stream/plugin/fields/ecs.yml delete mode 100644 packages/tenable_io/data_stream/plugin/fields/overridden-ecs.yml delete mode 100644 packages/tenable_io/data_stream/scan/fields/ecs.yml delete mode 100644 packages/tenable_io/data_stream/scan/fields/overridden-ecs.yml delete mode 100644 packages/tenable_io/data_stream/vulnerability/fields/ecs.yml delete mode 100644 packages/tenable_io/data_stream/vulnerability/fields/overridden-ecs.yml diff --git a/packages/tenable_io/changelog.yml b/packages/tenable_io/changelog.yml index 7fc7bc2157f..162283dac22 100644 --- a/packages/tenable_io/changelog.yml +++ b/packages/tenable_io/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.1.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "3.0.1" changes: - description: Resolved ignore_malformed issues with fields. diff --git a/packages/tenable_io/data_stream/asset/fields/agent.yml b/packages/tenable_io/data_stream/asset/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/tenable_io/data_stream/asset/fields/agent.yml +++ b/packages/tenable_io/data_stream/asset/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/tenable_io/data_stream/asset/fields/ecs.yml b/packages/tenable_io/data_stream/asset/fields/ecs.yml deleted file mode 100644 index 99ea44f80e6..00000000000 --- a/packages/tenable_io/data_stream/asset/fields/ecs.yml +++ /dev/null @@ -1,20 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.type -- external: ecs - name: message -- external: ecs - name: network.name -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: tags diff --git a/packages/tenable_io/data_stream/asset/fields/overridden-ecs.yml b/packages/tenable_io/data_stream/asset/fields/overridden-ecs.yml deleted file mode 100644 index 230ed31e27c..00000000000 --- a/packages/tenable_io/data_stream/asset/fields/overridden-ecs.yml +++ /dev/null @@ -1,4 +0,0 @@ -- name: event.original - type: keyword - ignore_above: 8191 - description: Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. diff --git a/packages/tenable_io/data_stream/plugin/fields/agent.yml b/packages/tenable_io/data_stream/plugin/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/tenable_io/data_stream/plugin/fields/agent.yml +++ b/packages/tenable_io/data_stream/plugin/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/tenable_io/data_stream/plugin/fields/ecs.yml b/packages/tenable_io/data_stream/plugin/fields/ecs.yml deleted file mode 100644 index f13d7301aa8..00000000000 --- a/packages/tenable_io/data_stream/plugin/fields/ecs.yml +++ /dev/null @@ -1,22 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.type -- external: ecs - name: message -- external: ecs - name: tags -- external: ecs - name: vulnerability.id -- external: ecs - name: vulnerability.reference -- external: ecs - name: vulnerability.scanner.vendor -- external: ecs - name: vulnerability.score.base -- external: ecs - name: vulnerability.score.temporal diff --git a/packages/tenable_io/data_stream/plugin/fields/overridden-ecs.yml b/packages/tenable_io/data_stream/plugin/fields/overridden-ecs.yml deleted file mode 100644 index 230ed31e27c..00000000000 --- a/packages/tenable_io/data_stream/plugin/fields/overridden-ecs.yml +++ /dev/null @@ -1,4 +0,0 @@ -- name: event.original - type: keyword - ignore_above: 8191 - description: Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. diff --git a/packages/tenable_io/data_stream/scan/fields/agent.yml b/packages/tenable_io/data_stream/scan/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/tenable_io/data_stream/scan/fields/agent.yml +++ b/packages/tenable_io/data_stream/scan/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/tenable_io/data_stream/scan/fields/ecs.yml b/packages/tenable_io/data_stream/scan/fields/ecs.yml deleted file mode 100644 index a9578688660..00000000000 --- a/packages/tenable_io/data_stream/scan/fields/ecs.yml +++ /dev/null @@ -1,18 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.type -- external: ecs - name: message -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: tags diff --git a/packages/tenable_io/data_stream/scan/fields/overridden-ecs.yml b/packages/tenable_io/data_stream/scan/fields/overridden-ecs.yml deleted file mode 100644 index 230ed31e27c..00000000000 --- a/packages/tenable_io/data_stream/scan/fields/overridden-ecs.yml +++ /dev/null @@ -1,4 +0,0 @@ -- name: event.original - type: keyword - ignore_above: 8191 - description: Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. diff --git a/packages/tenable_io/data_stream/vulnerability/fields/agent.yml b/packages/tenable_io/data_stream/vulnerability/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/tenable_io/data_stream/vulnerability/fields/agent.yml +++ b/packages/tenable_io/data_stream/vulnerability/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/tenable_io/data_stream/vulnerability/fields/ecs.yml b/packages/tenable_io/data_stream/vulnerability/fields/ecs.yml deleted file mode 100644 index 0468da3e2dc..00000000000 --- a/packages/tenable_io/data_stream/vulnerability/fields/ecs.yml +++ /dev/null @@ -1,40 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.type -- external: ecs - name: message -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: tags -- external: ecs - name: vulnerability.category -- external: ecs - name: vulnerability.classification -- external: ecs - name: vulnerability.enumeration -- external: ecs - name: vulnerability.id -- external: ecs - name: vulnerability.reference -- external: ecs - name: vulnerability.report_id -- external: ecs - name: vulnerability.scanner.vendor -- external: ecs - name: vulnerability.score.base -- external: ecs - name: vulnerability.score.version -- external: ecs - name: vulnerability.score.temporal -- external: ecs - name: vulnerability.severity diff --git a/packages/tenable_io/data_stream/vulnerability/fields/overridden-ecs.yml b/packages/tenable_io/data_stream/vulnerability/fields/overridden-ecs.yml deleted file mode 100644 index 230ed31e27c..00000000000 --- a/packages/tenable_io/data_stream/vulnerability/fields/overridden-ecs.yml +++ /dev/null @@ -1,4 +0,0 @@ -- name: event.original - type: keyword - ignore_above: 8191 - description: Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. diff --git a/packages/tenable_io/docs/README.md b/packages/tenable_io/docs/README.md index f9435a89039..b2984146ee7 100644 --- a/packages/tenable_io/docs/README.md +++ b/packages/tenable_io/docs/README.md @@ -257,54 +257,17 @@ An example event for `asset` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.name | Name given by operators to sections of their network. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| tags | List of keywords used to tag each event. | keyword | | tenable_io.asset.acr_score | The Asset Criticality Rating (ACR) for the asset. With Lumin, Tenable assigns an ACR to each asset on your network to represent the asset's relative risk as an integer from 1 to 10. | long | | tenable_io.asset.agent_names | The names of any Nessus agents that scanned and identified the asset. | keyword | | tenable_io.asset.agent_uuid | The unique identifier of the Nessus agent that identified the asset. | keyword | @@ -559,50 +522,17 @@ An example event for `plugin` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | | tenable_io.plugin.attributes.always_run | | boolean | | tenable_io.plugin.attributes.bid | | long | | tenable_io.plugin.attributes.compliance | | boolean | @@ -678,11 +608,6 @@ An example event for `plugin` looks as following: | tenable_io.plugin.attributes.xrefs.type | | keyword | | tenable_io.plugin.id | The ID of the plugin. | keyword | | tenable_io.plugin.name | The name of the plugin. | keyword | -| vulnerability.id | The identification (ID) is the number portion of a vulnerability entry. It includes a unique identification number for the vulnerability. For example (https://cve.mitre.org/about/faqs.html#what_is_cve_id)[Common Vulnerabilities and Exposure CVE ID] | keyword | -| vulnerability.reference | A resource that provides additional information, context, and mitigations for the identified vulnerability. | keyword | -| vulnerability.scanner.vendor | The name of the vulnerability scanner vendor. | keyword | -| vulnerability.score.base | Scores can range from 0.0 to 10.0, with 10.0 being the most severe. Base scores cover an assessment for exploitability metrics (attack vector, complexity, privileges, and user interaction), impact metrics (confidentiality, integrity, and availability), and scope. For example (https://www.first.org/cvss/specification-document) | float | -| vulnerability.score.temporal | Scores can range from 0.0 to 10.0, with 10.0 being the most severe. Temporal scores cover an assessment for code maturity, remediation level, and confidence. For example (https://www.first.org/cvss/specification-document) | float | ### vulnerability @@ -900,53 +825,17 @@ An example event for `vulnerability` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| tags | List of keywords used to tag each event. | keyword | | tenable_io.vulnerability.asset.agent_uuid | The UUID of the agent that performed the scan where the vulnerability was found. | keyword | | tenable_io.vulnerability.asset.bios_uuid | The BIOS UUID of the asset where the vulnerability was found. | keyword | | tenable_io.vulnerability.asset.device_type | The type of asset where the vulnerability was found. | keyword | @@ -1076,18 +965,7 @@ An example event for `vulnerability` looks as following: | tenable_io.vulnerability.severity.modification_type | The type of modification a user made to the vulnerability's severity. Possible values include:none, recasted and accepted. | keyword | | tenable_io.vulnerability.severity.value | The severity of the vulnerability as defined using the Common Vulnerability Scoring System (CVSS) base score. Possible values include info, low, medium, high and critical. | keyword | | tenable_io.vulnerability.state | The state of the vulnerability as determined by the Tenable Vulnerability Management state service. Possible values include: open, reopen and fixed. | keyword | -| vulnerability.category | The type of system or architecture that the vulnerability affects. These may be platform-specific (for example, Debian or SUSE) or general (for example, Database or Firewall). For example (https://qualysguard.qualys.com/qwebhelp/fo_portal/knowledgebase/vulnerability_categories.htm[Qualys vulnerability categories]) This field must be an array. | keyword | -| vulnerability.classification | The classification of the vulnerability scoring system. For example (https://www.first.org/cvss/) | keyword | | vulnerability.description | The description of the vulnerability. | text | -| vulnerability.enumeration | The type of identifier used for this vulnerability. For example (https://cve.mitre.org/about/) | keyword | -| vulnerability.id | The identification (ID) is the number portion of a vulnerability entry. It includes a unique identification number for the vulnerability. For example (https://cve.mitre.org/about/faqs.html#what_is_cve_id)[Common Vulnerabilities and Exposure CVE ID] | keyword | -| vulnerability.reference | A resource that provides additional information, context, and mitigations for the identified vulnerability. | keyword | -| vulnerability.report_id | The report or scan identification number. | keyword | -| vulnerability.scanner.vendor | The name of the vulnerability scanner vendor. | keyword | -| vulnerability.score.base | Scores can range from 0.0 to 10.0, with 10.0 being the most severe. Base scores cover an assessment for exploitability metrics (attack vector, complexity, privileges, and user interaction), impact metrics (confidentiality, integrity, and availability), and scope. For example (https://www.first.org/cvss/specification-document) | float | -| vulnerability.score.temporal | Scores can range from 0.0 to 10.0, with 10.0 being the most severe. Temporal scores cover an assessment for code maturity, remediation level, and confidence. For example (https://www.first.org/cvss/specification-document) | float | -| vulnerability.score.version | The National Vulnerability Database (NVD) provides qualitative severity rankings of "Low", "Medium", and "High" for CVSS v2.0 base score ranges in addition to the severity ratings for CVSS v3.0 as they are defined in the CVSS v3.0 specification. CVSS is owned and managed by FIRST.Org, Inc. (FIRST), a US-based non-profit organization, whose mission is to help computer security incident response teams across the world. For example (https://nvd.nist.gov/vuln-metrics/cvss) | keyword | -| vulnerability.severity | The severity of the vulnerability can help with metrics and internal prioritization regarding remediation. For example (https://nvd.nist.gov/vuln-metrics/cvss) | keyword | ### scan @@ -1186,53 +1064,17 @@ An example event for `scan` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| tags | List of keywords used to tag each event. | keyword | | tenable_io.scan.control | If true, the scan has a schedule and can be launched. | boolean | | tenable_io.scan.creation_date | For newly-created scans, the date on which the scan configuration was originally created. For scans that have been launched at least once, this attribute does not represent the date on which the scan configuration was originally created. Instead, it represents the date on which the scan was first launched, in Unix time format. | date | | tenable_io.scan.enabled | Indicates whether the scan schedule is active (true) or inactive (false). | boolean | diff --git a/packages/tenable_io/manifest.yml b/packages/tenable_io/manifest.yml index 0b113f79e0e..ba3dcad5cbe 100644 --- a/packages/tenable_io/manifest.yml +++ b/packages/tenable_io/manifest.yml @@ -1,14 +1,14 @@ format_version: "3.0.2" name: tenable_io title: Tenable Vulnerability Management -version: "3.0.1" +version: "3.1.0" description: Collect logs from Tenable Vulnerability Management with Elastic Agent. type: integration categories: - security conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" screenshots: - src: /img/tenable_io-screenshot.png title: Tenable Vulnerability Management dashboard screenshot From 5c469a2f47a1176325fa407127e2563d3f5d25b7 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:27:38 +0930 Subject: [PATCH 091/121] [tenable_sc] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/tenable_sc --- packages/tenable_sc/changelog.yml | 5 + .../data_stream/asset/fields/agent.yml | 147 ------------------ .../data_stream/asset/fields/ecs.yml | 16 -- .../data_stream/plugin/fields/agent.yml | 147 ------------------ .../data_stream/plugin/fields/ecs.yml | 14 -- .../vulnerability/fields/agent.yml | 147 ------------------ .../data_stream/vulnerability/fields/ecs.yml | 42 ----- packages/tenable_sc/docs/README.md | 115 -------------- packages/tenable_sc/manifest.yml | 4 +- 9 files changed, 7 insertions(+), 630 deletions(-) delete mode 100644 packages/tenable_sc/data_stream/asset/fields/ecs.yml delete mode 100644 packages/tenable_sc/data_stream/plugin/fields/ecs.yml delete mode 100644 packages/tenable_sc/data_stream/vulnerability/fields/ecs.yml diff --git a/packages/tenable_sc/changelog.yml b/packages/tenable_sc/changelog.yml index 0732cac4400..1f60cdb129c 100644 --- a/packages/tenable_sc/changelog.yml +++ b/packages/tenable_sc/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.23.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.22.0" changes: - description: Improve handling of empty responses. diff --git a/packages/tenable_sc/data_stream/asset/fields/agent.yml b/packages/tenable_sc/data_stream/asset/fields/agent.yml index 215021047d4..f833857d0fe 100644 --- a/packages/tenable_sc/data_stream/asset/fields/agent.yml +++ b/packages/tenable_sc/data_stream/asset/fields/agent.yml @@ -5,162 +5,15 @@ footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." - - name: id - level: core - type: keyword - ignore_above: 1024 - description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." - name: containerized type: boolean description: >- diff --git a/packages/tenable_sc/data_stream/asset/fields/ecs.yml b/packages/tenable_sc/data_stream/asset/fields/ecs.yml deleted file mode 100644 index 94317a5fa34..00000000000 --- a/packages/tenable_sc/data_stream/asset/fields/ecs.yml +++ /dev/null @@ -1,16 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.type -- external: ecs - name: related.ip -- external: ecs - name: related.hosts -- external: ecs - name: tags diff --git a/packages/tenable_sc/data_stream/plugin/fields/agent.yml b/packages/tenable_sc/data_stream/plugin/fields/agent.yml index 215021047d4..f833857d0fe 100644 --- a/packages/tenable_sc/data_stream/plugin/fields/agent.yml +++ b/packages/tenable_sc/data_stream/plugin/fields/agent.yml @@ -5,162 +5,15 @@ footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." - - name: id - level: core - type: keyword - ignore_above: 1024 - description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." - name: containerized type: boolean description: >- diff --git a/packages/tenable_sc/data_stream/plugin/fields/ecs.yml b/packages/tenable_sc/data_stream/plugin/fields/ecs.yml deleted file mode 100644 index 4aadb92e274..00000000000 --- a/packages/tenable_sc/data_stream/plugin/fields/ecs.yml +++ /dev/null @@ -1,14 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.type -- external: ecs - name: network.transport -- external: ecs - name: related.hash -- external: ecs - name: tags diff --git a/packages/tenable_sc/data_stream/vulnerability/fields/agent.yml b/packages/tenable_sc/data_stream/vulnerability/fields/agent.yml index 215021047d4..f833857d0fe 100644 --- a/packages/tenable_sc/data_stream/vulnerability/fields/agent.yml +++ b/packages/tenable_sc/data_stream/vulnerability/fields/agent.yml @@ -5,162 +5,15 @@ footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." - - name: id - level: core - type: keyword - ignore_above: 1024 - description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." - name: containerized type: boolean description: >- diff --git a/packages/tenable_sc/data_stream/vulnerability/fields/ecs.yml b/packages/tenable_sc/data_stream/vulnerability/fields/ecs.yml deleted file mode 100644 index 1388fea1271..00000000000 --- a/packages/tenable_sc/data_stream/vulnerability/fields/ecs.yml +++ /dev/null @@ -1,42 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.type -- external: ecs - name: network.transport -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: tags -- external: ecs - name: vulnerability.category -- external: ecs - name: vulnerability.classification -- external: ecs - name: vulnerability.description -- external: ecs - name: vulnerability.enumeration -- external: ecs - name: vulnerability.id -- external: ecs - name: vulnerability.severity -- external: ecs - name: vulnerability.reference -- external: ecs - name: vulnerability.report_id -- external: ecs - name: vulnerability.scanner.vendor -- external: ecs - name: vulnerability.score.base -- external: ecs - name: vulnerability.score.temporal -- external: ecs - name: vulnerability.score.version diff --git a/packages/tenable_sc/docs/README.md b/packages/tenable_sc/docs/README.md index 9a13e717bf4..f7732f9c08c 100644 --- a/packages/tenable_sc/docs/README.md +++ b/packages/tenable_sc/docs/README.md @@ -143,51 +143,17 @@ An example event for `asset` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| tags | List of keywords used to tag each event. | keyword | | tenable_sc.asset.bios.guid | GUID of bios. | keyword | | tenable_sc.asset.custom_hash | Hash representing the values of the field names mentioned in uniqueness field in order to uniquely identify an asset. | keyword | | tenable_sc.asset.dns.name | DNS name of the asset. | keyword | @@ -388,50 +354,17 @@ An example event for `plugin` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| tags | List of keywords used to tag each event. | keyword | | tenable_sc.plugin.base_score | The CVSSv2 base score (intrinsic and fundamental characteristics of a vulnerability that are constant over time and user environments). | double | | tenable_sc.plugin.check_type | The type of the compliance check that detected the vulnerability. | keyword | | tenable_sc.plugin.copyright | The copyright information related to the plugin. | keyword | @@ -715,52 +648,17 @@ An example event for `vulnerability` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| tags | List of keywords used to tag each event. | keyword | | tenable_sc.vulnerability.accept_risk | N/A. | keyword | | tenable_sc.vulnerability.age | The time in days between the first and last time the vulnerability was seen. | long | | tenable_sc.vulnerability.base_score | Intrinsic and fundamental characteristics of a vulnerability that are constant over time and user environments. | keyword | @@ -819,16 +717,3 @@ An example event for `vulnerability` looks as following: | tenable_sc.vulnerability.vpr.score | The Vulnerability Priority Rating (VPR) score for the vulnerability. | double | | tenable_sc.vulnerability.vuln_pub_date | The date on which the vulnerability was published. | date | | tenable_sc.vulnerability.xref | References to third-party information about the vulnerability, exploit, or update associated with the plugin. | keyword | -| vulnerability.category | The type of system or architecture that the vulnerability affects. These may be platform-specific (for example, Debian or SUSE) or general (for example, Database or Firewall). For example (https://qualysguard.qualys.com/qwebhelp/fo_portal/knowledgebase/vulnerability_categories.htm[Qualys vulnerability categories]) This field must be an array. | keyword | -| vulnerability.classification | The classification of the vulnerability scoring system. For example (https://www.first.org/cvss/) | keyword | -| vulnerability.description | The description of the vulnerability that provides additional context of the vulnerability. For example (https://cve.mitre.org/about/faqs.html#cve_entry_descriptions_created[Common Vulnerabilities and Exposure CVE description]) | keyword | -| vulnerability.description.text | Multi-field of `vulnerability.description`. | match_only_text | -| vulnerability.enumeration | The type of identifier used for this vulnerability. For example (https://cve.mitre.org/about/) | keyword | -| vulnerability.id | The identification (ID) is the number portion of a vulnerability entry. It includes a unique identification number for the vulnerability. For example (https://cve.mitre.org/about/faqs.html#what_is_cve_id)[Common Vulnerabilities and Exposure CVE ID] | keyword | -| vulnerability.reference | A resource that provides additional information, context, and mitigations for the identified vulnerability. | keyword | -| vulnerability.report_id | The report or scan identification number. | keyword | -| vulnerability.scanner.vendor | The name of the vulnerability scanner vendor. | keyword | -| vulnerability.score.base | Scores can range from 0.0 to 10.0, with 10.0 being the most severe. Base scores cover an assessment for exploitability metrics (attack vector, complexity, privileges, and user interaction), impact metrics (confidentiality, integrity, and availability), and scope. For example (https://www.first.org/cvss/specification-document) | float | -| vulnerability.score.temporal | Scores can range from 0.0 to 10.0, with 10.0 being the most severe. Temporal scores cover an assessment for code maturity, remediation level, and confidence. For example (https://www.first.org/cvss/specification-document) | float | -| vulnerability.score.version | The National Vulnerability Database (NVD) provides qualitative severity rankings of "Low", "Medium", and "High" for CVSS v2.0 base score ranges in addition to the severity ratings for CVSS v3.0 as they are defined in the CVSS v3.0 specification. CVSS is owned and managed by FIRST.Org, Inc. (FIRST), a US-based non-profit organization, whose mission is to help computer security incident response teams across the world. For example (https://nvd.nist.gov/vuln-metrics/cvss) | keyword | -| vulnerability.severity | The severity of the vulnerability can help with metrics and internal prioritization regarding remediation. For example (https://nvd.nist.gov/vuln-metrics/cvss) | keyword | diff --git a/packages/tenable_sc/manifest.yml b/packages/tenable_sc/manifest.yml index a5c963af178..0c0fb8891b8 100644 --- a/packages/tenable_sc/manifest.yml +++ b/packages/tenable_sc/manifest.yml @@ -2,7 +2,7 @@ format_version: "3.0.2" name: tenable_sc title: Tenable.sc # The version must be updated in the input configuration templates as well, in order to set the correct User-Agent header. Until elastic/kibana#121310 is implemented we will have to manually sync these. -version: "1.22.0" +version: "1.23.0" description: | Collect logs from Tenable.sc with Elastic Agent. type: integration @@ -10,7 +10,7 @@ categories: - security conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" screenshots: - src: /img/tenable_sc-screenshot.png title: Tenable.sc vulnerability dashboard screenshot From e0e2387d7089e254654a7e06764d5f5111200a2e Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:27:39 +0930 Subject: [PATCH 092/121] [thycotic_ss] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.5.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/thycotic_ss --- packages/thycotic_ss/changelog.yml | 5 ++ .../data_stream/logs/fields/ecs.yml | 88 ------------------- packages/thycotic_ss/docs/README.md | 47 ---------- packages/thycotic_ss/manifest.yml | 4 +- 4 files changed, 7 insertions(+), 137 deletions(-) diff --git a/packages/thycotic_ss/changelog.yml b/packages/thycotic_ss/changelog.yml index 62c79def1ad..ee6e591369a 100644 --- a/packages/thycotic_ss/changelog.yml +++ b/packages/thycotic_ss/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.8.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.7.0" changes: - description: Update manifest format version to v3.0.3. diff --git a/packages/thycotic_ss/data_stream/logs/fields/ecs.yml b/packages/thycotic_ss/data_stream/logs/fields/ecs.yml index 38d7ca618aa..adb0dc85322 100644 --- a/packages/thycotic_ss/data_stream/logs/fields/ecs.yml +++ b/packages/thycotic_ss/data_stream/logs/fields/ecs.yml @@ -1,90 +1,2 @@ - external: ecs name: '@timestamp' -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.code -- external: ecs - name: event.ingested -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.timezone -- external: ecs - name: event.type -- external: ecs - name: event.category -- external: ecs - name: event.kind -- external: ecs - name: group.id -- external: ecs - name: group.name -- external: ecs - name: host.hostname -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: log.level -- external: ecs - name: log.syslog.facility.code -- external: ecs - name: log.syslog.priority -- external: ecs - name: log.syslog.severity.code -- external: ecs - name: message -- external: ecs - name: observer.product -- external: ecs - name: observer.type -- external: ecs - name: observer.vendor -- external: ecs - name: observer.version -- external: ecs - name: observer.hostname -- external: ecs - name: observer.ip -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: source.address -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.ip -- external: ecs - name: source.mac -- external: ecs - name: tags -- external: ecs - name: user.domain -- external: ecs - name: user.full_name -- external: ecs - name: user.id -- external: ecs - name: user.name diff --git a/packages/thycotic_ss/docs/README.md b/packages/thycotic_ss/docs/README.md index 57ad649f44a..3ae9890cde4 100644 --- a/packages/thycotic_ss/docs/README.md +++ b/packages/thycotic_ss/docs/README.md @@ -190,48 +190,7 @@ The following fields may be used by the package: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| group.id | Unique identifier for the group on the system/platform. | keyword | -| group.name | Name of the group. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | input.type | | keyword | -| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | -| log.syslog.facility.code | The Syslog numeric facility of the log event, if available. According to RFCs 5424 and 3164, this value should be an integer between 0 and 23. | long | -| log.syslog.priority | Syslog numeric priority of the event, if available. According to RFCs 5424 and 3164, the priority is 8 \* facility + severity. This number is therefore expected to contain a value between 0 and 191. | long | -| log.syslog.severity.code | The Syslog numeric severity of the log event, if available. If the event source publishing via Syslog provides a different numeric severity value (e.g. firewall, IDS), your source's numeric severity should go to `event.severity`. If the event source does not specify a distinct severity, you can optionally copy the Syslog severity to `event.severity`. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| observer.hostname | Hostname of the observer. | keyword | -| observer.ip | IP addresses of the observer. | ip | -| observer.product | The product name of the observer. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| observer.version | Observer version. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.mac | MAC address of the source. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| tags | List of keywords used to tag each event. | keyword | | thycotic_ss.event.folder.folder | | keyword | | thycotic_ss.event.folder.id | | keyword | | thycotic_ss.event.folder.name | | keyword | @@ -253,10 +212,4 @@ The following fields may be used by the package: | thycotic_ss.event.user.full_name | | keyword | | thycotic_ss.event.user.id | | keyword | | thycotic_ss.event.user.name | | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | diff --git a/packages/thycotic_ss/manifest.yml b/packages/thycotic_ss/manifest.yml index e28026f8f65..f786874bda6 100644 --- a/packages/thycotic_ss/manifest.yml +++ b/packages/thycotic_ss/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: thycotic_ss title: "Thycotic Secret Server" -version: "1.7.0" +version: "1.8.0" source: license: "Elastic-2.0" description: "Thycotic Secret Server logs" @@ -10,7 +10,7 @@ categories: - security conditions: kibana: - version: "^8.5.0" + version: "^8.13.0" elastic: subscription: "basic" screenshots: From 5819d1e8c498aef47de744520be5578b63594834 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:27:44 +0930 Subject: [PATCH 093/121] [ti_abusech] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/ti_abusech --- packages/ti_abusech/changelog.yml | 5 + .../data_stream/malware/fields/agent.yml | 167 +----------- .../data_stream/malware/fields/beats.yml | 3 - .../data_stream/malware/fields/ecs.yml | 47 ---- .../malwarebazaar/fields/agent.yml | 167 +----------- .../malwarebazaar/fields/beats.yml | 3 - .../data_stream/malwarebazaar/fields/ecs.yml | 78 ------ .../data_stream/threatfox/fields/agent.yml | 167 +----------- .../data_stream/threatfox/fields/beats.yml | 3 - .../data_stream/threatfox/fields/ecs.yml | 88 ------- .../test-abusechurl-dump.log-expected.json | 8 +- .../data_stream/url/fields/agent.yml | 167 +----------- .../data_stream/url/fields/beats.yml | 3 - .../ti_abusech/data_stream/url/fields/ecs.yml | 54 ---- packages/ti_abusech/docs/README.md | 243 ------------------ .../transform/latest_malware/fields/ecs.yml | 2 +- .../latest_malwarebazaar/fields/ecs.yml | 2 +- .../transform/latest_url/fields/ecs.yml | 2 +- packages/ti_abusech/manifest.yml | 4 +- 19 files changed, 18 insertions(+), 1195 deletions(-) delete mode 100644 packages/ti_abusech/data_stream/malware/fields/ecs.yml delete mode 100644 packages/ti_abusech/data_stream/malwarebazaar/fields/ecs.yml delete mode 100644 packages/ti_abusech/data_stream/threatfox/fields/ecs.yml delete mode 100644 packages/ti_abusech/data_stream/url/fields/ecs.yml diff --git a/packages/ti_abusech/changelog.yml b/packages/ti_abusech/changelog.yml index ba3fd5c699a..eaf3227c855 100644 --- a/packages/ti_abusech/changelog.yml +++ b/packages/ti_abusech/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.2.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.1.0" changes: - description: Improve error handling and reporting in malwarebazaar data stream. diff --git a/packages/ti_abusech/data_stream/malware/fields/agent.yml b/packages/ti_abusech/data_stream/malware/fields/agent.yml index da4e652c53b..2bc58530bac 100644 --- a/packages/ti_abusech/data_stream/malware/fields/agent.yml +++ b/packages/ti_abusech/data_stream/malware/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/ti_abusech/data_stream/malware/fields/beats.yml b/packages/ti_abusech/data_stream/malware/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/ti_abusech/data_stream/malware/fields/beats.yml +++ b/packages/ti_abusech/data_stream/malware/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/ti_abusech/data_stream/malware/fields/ecs.yml b/packages/ti_abusech/data_stream/malware/fields/ecs.yml deleted file mode 100644 index bb08d74e4bb..00000000000 --- a/packages/ti_abusech/data_stream/malware/fields/ecs.yml +++ /dev/null @@ -1,47 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: error.message -- external: ecs - name: tags -- external: ecs - name: related.hash -- external: ecs - name: event.created -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.category -- external: ecs - name: event.type -- external: ecs - name: event.original -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.first_seen -- external: ecs - name: threat.indicator.file.size -- external: ecs - name: threat.indicator.file.type -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.file.pe.imphash -- external: ecs - name: threat.indicator.file.hash.ssdeep -- name: threat.indicator.file.hash.tlsh - type: keyword - description: "The file's import tlsh, if available." -- name: threat.indicator.provider - external: ecs -- name: threat.indicator.name - external: ecs -- name: labels - external: ecs diff --git a/packages/ti_abusech/data_stream/malwarebazaar/fields/agent.yml b/packages/ti_abusech/data_stream/malwarebazaar/fields/agent.yml index da4e652c53b..2bc58530bac 100644 --- a/packages/ti_abusech/data_stream/malwarebazaar/fields/agent.yml +++ b/packages/ti_abusech/data_stream/malwarebazaar/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/ti_abusech/data_stream/malwarebazaar/fields/beats.yml b/packages/ti_abusech/data_stream/malwarebazaar/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/ti_abusech/data_stream/malwarebazaar/fields/beats.yml +++ b/packages/ti_abusech/data_stream/malwarebazaar/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/ti_abusech/data_stream/malwarebazaar/fields/ecs.yml b/packages/ti_abusech/data_stream/malwarebazaar/fields/ecs.yml deleted file mode 100644 index cc05f196864..00000000000 --- a/packages/ti_abusech/data_stream/malwarebazaar/fields/ecs.yml +++ /dev/null @@ -1,78 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: error.message -- external: ecs - name: tags -- external: ecs - name: related.hash -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.category -- external: ecs - name: event.type -- external: ecs - name: event.original -- external: ecs - name: threat.indicator.type -- external: ecs - name: event.created -- external: ecs - name: threat.indicator.first_seen -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.indicator.file.size -- external: ecs - name: threat.indicator.file.type -- external: ecs - name: threat.indicator.file.name -- external: ecs - name: threat.indicator.file.extension -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.mime_type -- external: ecs - name: threat.software.alias -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.file.hash.ssdeep -- name: threat.indicator.file.hash.sha384 - type: keyword - description: "The file's sha384 hash, if available." -- name: threat.indicator.file.hash.tlsh - type: keyword - description: "The file's import tlsh, if available." -- external: ecs - name: threat.indicator.file.pe.imphash -- external: ecs - name: threat.indicator.file.elf.telfhash -- name: threat.indicator.file.x509.subject.common_name - external: ecs -- name: threat.indicator.file.x509.issuer.common_name - external: ecs -- name: threat.indicator.file.x509.public_key_algorithm - external: ecs -- name: threat.indicator.file.x509.not_before - external: ecs -- name: threat.indicator.file.x509.not_after - external: ecs -- name: threat.indicator.file.x509.serial_number - external: ecs -- name: threat.indicator.provider - external: ecs -- name: threat.indicator.geo.country_iso_code - external: ecs -- name: threat.indicator.name - external: ecs -- name: labels - external: ecs diff --git a/packages/ti_abusech/data_stream/threatfox/fields/agent.yml b/packages/ti_abusech/data_stream/threatfox/fields/agent.yml index da4e652c53b..2bc58530bac 100644 --- a/packages/ti_abusech/data_stream/threatfox/fields/agent.yml +++ b/packages/ti_abusech/data_stream/threatfox/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/ti_abusech/data_stream/threatfox/fields/beats.yml b/packages/ti_abusech/data_stream/threatfox/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/ti_abusech/data_stream/threatfox/fields/beats.yml +++ b/packages/ti_abusech/data_stream/threatfox/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/ti_abusech/data_stream/threatfox/fields/ecs.yml b/packages/ti_abusech/data_stream/threatfox/fields/ecs.yml deleted file mode 100644 index e994b9dc5aa..00000000000 --- a/packages/ti_abusech/data_stream/threatfox/fields/ecs.yml +++ /dev/null @@ -1,88 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: error.message -- external: ecs - name: tags -- external: ecs - name: related.hash -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.category -- external: ecs - name: event.type -- external: ecs - name: event.original -- external: ecs - name: threat.indicator.type -- external: ecs - name: event.created -- external: ecs - name: threat.indicator.first_seen -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.indicator.file.size -- external: ecs - name: threat.indicator.file.type -- external: ecs - name: threat.indicator.file.name -- external: ecs - name: threat.indicator.file.extension -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.mime_type -- external: ecs - name: threat.software.alias -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.file.hash.ssdeep -- external: ecs - name: threat.indicator.file.hash.sha384 -- external: ecs - name: threat.indicator.file.hash.tlsh -- external: ecs - name: threat.indicator.file.pe.imphash -- external: ecs - name: threat.indicator.file.elf.telfhash -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.port -- external: ecs - name: threat.indicator.ip -- external: ecs - name: threat.indicator.confidence -- external: ecs - name: threat.indicator.description -- external: ecs - name: threat.indicator.reference -- external: ecs - name: threat.indicator.url.domain -- external: ecs - name: threat.indicator.url.extension -- external: ecs - name: threat.indicator.url.original -- external: ecs - name: threat.indicator.url.path -- external: ecs - name: threat.indicator.url.port -- external: ecs - name: threat.indicator.url.scheme -- external: ecs - name: threat.software.name -- external: ecs - name: threat.software.reference -- external: ecs - name: threat.indicator.name -- external: ecs - name: labels diff --git a/packages/ti_abusech/data_stream/url/_dev/test/pipeline/test-abusechurl-dump.log-expected.json b/packages/ti_abusech/data_stream/url/_dev/test/pipeline/test-abusechurl-dump.log-expected.json index 33a3cc03f15..e043fd8a27a 100644 --- a/packages/ti_abusech/data_stream/url/_dev/test/pipeline/test-abusechurl-dump.log-expected.json +++ b/packages/ti_abusech/data_stream/url/_dev/test/pipeline/test-abusechurl-dump.log-expected.json @@ -3,7 +3,7 @@ { "abusech": { "url": { - "deleted_at": "2024-04-17T16:09:49.442Z", + "deleted_at": "2024-06-20T04:56:13.743Z", "id": "2786904", "threat": "malware_download", "url_status": "online" @@ -16,7 +16,7 @@ "category": [ "threat" ], - "ingested": "2024-04-17T15:11:19.442760096Z", + "ingested": "2024-06-20T03:57:43.743250436Z", "kind": "enrichment", "original": "{\"id\":\"2786904\",\"dateadded\":\"2024-03-19 11:34:09 UTC\",\"url\":\"http://115.55.244.160:41619/Mozi.m\",\"url_status\":\"online\",\"last_online\":\"2024-03-19 11:34:09 UTC\",\"threat\":\"malware_download\",\"tags\":[\"elf\",\"Mozi\"],\"urlhaus_link\":\"https://urlhaus.abuse.ch/url/2786904/\",\"reporter\":\"lrz_urlhaus\"}", "type": [ @@ -54,7 +54,7 @@ { "abusech": { "url": { - "deleted_at": "2024-04-17T16:09:49.442Z", + "deleted_at": "2024-06-20T04:56:13.743Z", "id": "2786903", "threat": "malware_download", "url_status": "online" @@ -67,7 +67,7 @@ "category": [ "threat" ], - "ingested": "2024-04-17T15:11:19.442771804Z", + "ingested": "2024-06-20T03:57:43.743264045Z", "kind": "enrichment", "original": "{\"id\":\"2786903\",\"dateadded\":\"2024-03-19 11:33:08 UTC\",\"url\":\"http://27.206.236.188:59429/i\",\"url_status\":\"online\",\"last_online\":\"2024-03-19 11:33:08 UTC\",\"threat\":\"malware_download\",\"tags\":[\"32-bit\",\"elf\",\"mips\",\"Mozi\"],\"urlhaus_link\":\"https://urlhaus.abuse.ch/url/2786903/\",\"reporter\":\"geenensp\"}", "type": [ diff --git a/packages/ti_abusech/data_stream/url/fields/agent.yml b/packages/ti_abusech/data_stream/url/fields/agent.yml index da4e652c53b..2bc58530bac 100644 --- a/packages/ti_abusech/data_stream/url/fields/agent.yml +++ b/packages/ti_abusech/data_stream/url/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/ti_abusech/data_stream/url/fields/beats.yml b/packages/ti_abusech/data_stream/url/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/ti_abusech/data_stream/url/fields/beats.yml +++ b/packages/ti_abusech/data_stream/url/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/ti_abusech/data_stream/url/fields/ecs.yml b/packages/ti_abusech/data_stream/url/fields/ecs.yml deleted file mode 100644 index a79e7924260..00000000000 --- a/packages/ti_abusech/data_stream/url/fields/ecs.yml +++ /dev/null @@ -1,54 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: error.message -- external: ecs - name: tags -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.category -- external: ecs - name: event.type -- external: ecs - name: event.created -- external: ecs - name: event.original -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.reference -- external: ecs - name: threat.indicator.first_seen -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.indicator.url.domain -- external: ecs - name: threat.indicator.url.full -- external: ecs - name: threat.indicator.url.fragment -- external: ecs - name: threat.indicator.url.extension -- external: ecs - name: threat.indicator.url.original -- external: ecs - name: threat.indicator.url.path -- external: ecs - name: threat.indicator.url.port -- external: ecs - name: threat.indicator.url.scheme -- external: ecs - name: threat.indicator.url.query -- external: ecs - name: threat.indicator.ip -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.name -- external: ecs - name: labels diff --git a/packages/ti_abusech/docs/README.md b/packages/ti_abusech/docs/README.md index 3f76752f36e..dffed3b7c6d 100644 --- a/packages/ti_abusech/docs/README.md +++ b/packages/ti_abusech/docs/README.md @@ -43,78 +43,22 @@ The AbuseCH URL data_stream retrieves full list of active threat intelligence in | abusech.url.threat | The threat corresponding to this malware URL. | keyword | | abusech.url.url_status | The current status of the URL. Possible values are: online, offline and unknown. | keyword | | abusech.url.urlhaus_reference | Link to URLhaus entry. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| labels | Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels. | object | | labels.interval | User-configured value for `Interval` setting. This is used in calculation of indicator expiration time. | keyword | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | | threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword | | threat.feed.name | Display friendly feed name | constant_keyword | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.name | The display name indicator in an UI friendly format URL, IP address, email address, registry key, port number, hash value, or other relevant name can serve as the display name. | keyword | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.reference | Reference URL linking to additional information about this indicator. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| threat.indicator.url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | -| threat.indicator.url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| threat.indicator.url.full.text | Multi-field of `threat.indicator.url.full`. | match_only_text | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | -| threat.indicator.url.path | Path of the request, such as "/search". | wildcard | -| threat.indicator.url.port | Port of the request, such as 443. | long | -| threat.indicator.url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| threat.indicator.url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | ### Malware @@ -132,71 +76,21 @@ The AbuseCH malware data_stream retrieves threat intelligence indicators from th | abusech.malware.virustotal.link | Link to the Virustotal report. | keyword | | abusech.malware.virustotal.percent | AV detection in percent. | float | | abusech.malware.virustotal.result | AV detection ratio. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| labels | Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels. | object | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| tags | List of keywords used to tag each event. | keyword | | threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword | | threat.feed.name | Display friendly feed name | constant_keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.file.hash.ssdeep | SSDEEP hash. | keyword | -| threat.indicator.file.hash.tlsh | The file's import tlsh, if available. | keyword | -| threat.indicator.file.pe.imphash | A hash of the imports in a PE file. An imphash -- or import hash -- can be used to fingerprint binaries even after recompilation or other code-level transformations have occurred, which would change more traditional hash values. Learn more at https://www.fireeye.com/blog/threat-research/2014/01/tracking-malware-import-hashing.html. | keyword | -| threat.indicator.file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| threat.indicator.file.type | File type (file, dir, or symlink). | keyword | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.name | The display name indicator in an UI friendly format URL, IP address, email address, registry key, port number, hash value, or other relevant name can serve as the display name. | keyword | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | ### MalwareBazaar @@ -226,86 +120,21 @@ The AbuseCH malwarebazaar data_stream retrieves threat intelligence indicators f | abusech.malwarebazaar.intelligence.mail.IT | Malware seen in IT spam traffic. | keyword | | abusech.malwarebazaar.intelligence.uploads | Number of uploads from MalwareBazaar. | long | | abusech.malwarebazaar.ioc_expiration_duration | The configured expiration duration. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| labels | Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels. | object | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| tags | List of keywords used to tag each event. | keyword | | threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword | | threat.feed.name | Display friendly feed name | constant_keyword | -| threat.indicator.file.elf.telfhash | telfhash symbol hash for ELF file. | keyword | -| threat.indicator.file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.file.hash.sha384 | The file's sha384 hash, if available. | keyword | -| threat.indicator.file.hash.ssdeep | SSDEEP hash. | keyword | -| threat.indicator.file.hash.tlsh | The file's import tlsh, if available. | keyword | -| threat.indicator.file.mime_type | MIME type should identify the format of the file or stream of bytes using https://www.iana.org/assignments/media-types/media-types.xhtml[IANA official types], where possible. When more than one type is applicable, the most specific type should be used. | keyword | -| threat.indicator.file.name | Name of the file including the extension, without the directory. | keyword | -| threat.indicator.file.pe.imphash | A hash of the imports in a PE file. An imphash -- or import hash -- can be used to fingerprint binaries even after recompilation or other code-level transformations have occurred, which would change more traditional hash values. Learn more at https://www.fireeye.com/blog/threat-research/2014/01/tracking-malware-import-hashing.html. | keyword | -| threat.indicator.file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| threat.indicator.file.type | File type (file, dir, or symlink). | keyword | -| threat.indicator.file.x509.issuer.common_name | List of common name (CN) of issuing certificate authority. | keyword | -| threat.indicator.file.x509.not_after | Time at which the certificate is no longer considered valid. | date | -| threat.indicator.file.x509.not_before | Time at which the certificate is first considered valid. | date | -| threat.indicator.file.x509.public_key_algorithm | Algorithm used to generate the public key. | keyword | -| threat.indicator.file.x509.serial_number | Unique serial number issued by the certificate authority. For consistency, if this value is alphanumeric, it should be formatted without colons and uppercase characters. | keyword | -| threat.indicator.file.x509.subject.common_name | List of common names (CN) of subject. | keyword | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.geo.country_iso_code | Country ISO code. | keyword | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.name | The display name indicator in an UI friendly format URL, IP address, email address, registry key, port number, hash value, or other relevant name can serve as the display name. | keyword | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.software.alias | The alias(es) of the software for a set of related intrusion activity that are tracked by a common name in the security community. While not required, you can use a MITRE ATT&CK® associated software description. | keyword | ### Threat Fox @@ -324,90 +153,18 @@ The AbuseCH threatfox data_stream retrieves threat intelligence indicators from | abusech.threatfox.tags | A list of tags associated with the queried malware sample. | keyword | | abusech.threatfox.threat_type | The type of threat. | keyword | | abusech.threatfox.threat_type_desc | The threat descsription. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| labels | Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels. | object | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| tags | List of keywords used to tag each event. | keyword | | threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword | | threat.feed.name | Display friendly feed name | constant_keyword | -| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword | -| threat.indicator.description | Describes the type of action conducted by the threat. | keyword | -| threat.indicator.file.elf.telfhash | telfhash symbol hash for ELF file. | keyword | -| threat.indicator.file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.file.hash.sha384 | SHA384 hash. | keyword | -| threat.indicator.file.hash.ssdeep | SSDEEP hash. | keyword | -| threat.indicator.file.hash.tlsh | TLSH hash. | keyword | -| threat.indicator.file.mime_type | MIME type should identify the format of the file or stream of bytes using https://www.iana.org/assignments/media-types/media-types.xhtml[IANA official types], where possible. When more than one type is applicable, the most specific type should be used. | keyword | -| threat.indicator.file.name | Name of the file including the extension, without the directory. | keyword | -| threat.indicator.file.pe.imphash | A hash of the imports in a PE file. An imphash -- or import hash -- can be used to fingerprint binaries even after recompilation or other code-level transformations have occurred, which would change more traditional hash values. Learn more at https://www.fireeye.com/blog/threat-research/2014/01/tracking-malware-import-hashing.html. | keyword | -| threat.indicator.file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| threat.indicator.file.type | File type (file, dir, or symlink). | keyword | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.name | The display name indicator in an UI friendly format URL, IP address, email address, registry key, port number, hash value, or other relevant name can serve as the display name. | keyword | -| threat.indicator.port | Identifies a threat indicator as a port number (irrespective of direction). | long | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.reference | Reference URL linking to additional information about this indicator. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | -| threat.indicator.url.path | Path of the request, such as "/search". | wildcard | -| threat.indicator.url.port | Port of the request, such as 443. | long | -| threat.indicator.url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| threat.software.alias | The alias(es) of the software for a set of related intrusion activity that are tracked by a common name in the security community. While not required, you can use a MITRE ATT&CK® associated software description. | keyword | -| threat.software.name | The name of the software used by this threat to conduct behavior commonly modeled using MITRE ATT&CK®. While not required, you can use a MITRE ATT&CK® software name. | keyword | -| threat.software.reference | The reference URL of the software used by this threat to conduct behavior commonly modeled using MITRE ATT&CK®. While not required, you can use a MITRE ATT&CK® software reference URL. | keyword | diff --git a/packages/ti_abusech/elasticsearch/transform/latest_malware/fields/ecs.yml b/packages/ti_abusech/elasticsearch/transform/latest_malware/fields/ecs.yml index f2d8810b7f8..341ded8e127 100644 --- a/packages/ti_abusech/elasticsearch/transform/latest_malware/fields/ecs.yml +++ b/packages/ti_abusech/elasticsearch/transform/latest_malware/fields/ecs.yml @@ -62,4 +62,4 @@ - name: threat.feed.dashboard_id type: constant_keyword description: Dashboard ID used for Kibana CTI UI - value: ti_abusech-c0d8d1f0-3b20-11ec-ae50-2fdf1e96c6a6 \ No newline at end of file + value: ti_abusech-c0d8d1f0-3b20-11ec-ae50-2fdf1e96c6a6 diff --git a/packages/ti_abusech/elasticsearch/transform/latest_malwarebazaar/fields/ecs.yml b/packages/ti_abusech/elasticsearch/transform/latest_malwarebazaar/fields/ecs.yml index b20161a264d..b8e179e3488 100644 --- a/packages/ti_abusech/elasticsearch/transform/latest_malwarebazaar/fields/ecs.yml +++ b/packages/ti_abusech/elasticsearch/transform/latest_malwarebazaar/fields/ecs.yml @@ -93,4 +93,4 @@ - name: threat.feed.dashboard_id type: constant_keyword description: Dashboard ID used for Kibana CTI UI - value: ti_abusech-c0d8d1f0-3b20-11ec-ae50-2fdf1e96c6a6 \ No newline at end of file + value: ti_abusech-c0d8d1f0-3b20-11ec-ae50-2fdf1e96c6a6 diff --git a/packages/ti_abusech/elasticsearch/transform/latest_url/fields/ecs.yml b/packages/ti_abusech/elasticsearch/transform/latest_url/fields/ecs.yml index d2e06310dbd..53b57d765b4 100644 --- a/packages/ti_abusech/elasticsearch/transform/latest_url/fields/ecs.yml +++ b/packages/ti_abusech/elasticsearch/transform/latest_url/fields/ecs.yml @@ -69,4 +69,4 @@ - name: threat.feed.dashboard_id type: constant_keyword description: Dashboard ID used for Kibana CTI UI - value: ti_abusech-c0d8d1f0-3b20-11ec-ae50-2fdf1e96c6a6 \ No newline at end of file + value: ti_abusech-c0d8d1f0-3b20-11ec-ae50-2fdf1e96c6a6 diff --git a/packages/ti_abusech/manifest.yml b/packages/ti_abusech/manifest.yml index d4570c22f39..361d540c292 100644 --- a/packages/ti_abusech/manifest.yml +++ b/packages/ti_abusech/manifest.yml @@ -1,13 +1,13 @@ name: ti_abusech title: AbuseCH -version: "2.1.0" +version: "2.2.0" description: Ingest threat intelligence indicators from URL Haus, Malware Bazaar, and Threat Fox feeds with Elastic Agent. type: integration format_version: "3.0.3" categories: ["security", "threat_intel"] conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" icons: - src: /img/abusech2.svg title: AbuseCH From 1e4184c79f8fe642923da72595fd82957cf4bb79 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:27:46 +0930 Subject: [PATCH 094/121] [ti_anomali] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/ti_anomali --- packages/ti_anomali/changelog.yml | 5 + .../data_stream/threatstream/fields/agent.yml | 167 +----------------- .../data_stream/threatstream/fields/beats.yml | 3 - .../data_stream/threatstream/fields/ecs.yml | 72 -------- packages/ti_anomali/docs/README.md | 66 ------- packages/ti_anomali/manifest.yml | 4 +- 6 files changed, 8 insertions(+), 309 deletions(-) delete mode 100644 packages/ti_anomali/data_stream/threatstream/fields/ecs.yml diff --git a/packages/ti_anomali/changelog.yml b/packages/ti_anomali/changelog.yml index e9dec6c8ef0..fe2cd13c3ae 100644 --- a/packages/ti_anomali/changelog.yml +++ b/packages/ti_anomali/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.22.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.21.0" changes: - description: Add destination index alias and fix docs. diff --git a/packages/ti_anomali/data_stream/threatstream/fields/agent.yml b/packages/ti_anomali/data_stream/threatstream/fields/agent.yml index da4e652c53b..2bc58530bac 100644 --- a/packages/ti_anomali/data_stream/threatstream/fields/agent.yml +++ b/packages/ti_anomali/data_stream/threatstream/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/ti_anomali/data_stream/threatstream/fields/beats.yml b/packages/ti_anomali/data_stream/threatstream/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/ti_anomali/data_stream/threatstream/fields/beats.yml +++ b/packages/ti_anomali/data_stream/threatstream/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/ti_anomali/data_stream/threatstream/fields/ecs.yml b/packages/ti_anomali/data_stream/threatstream/fields/ecs.yml deleted file mode 100644 index dc643b95903..00000000000 --- a/packages/ti_anomali/data_stream/threatstream/fields/ecs.yml +++ /dev/null @@ -1,72 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: tags -- external: ecs - name: error.message -- external: ecs - name: event.severity -- external: ecs - name: event.category -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.type -- external: ecs - name: event.created -- external: ecs - name: event.original -- external: ecs - name: threat.indicator.first_seen -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.ip -- external: ecs - name: threat.indicator.url.domain -- external: ecs - name: threat.indicator.url.full -- external: ecs - name: threat.indicator.url.extension -- external: ecs - name: threat.indicator.url.original -- external: ecs - name: threat.indicator.url.path -- external: ecs - name: threat.indicator.url.port -- external: ecs - name: threat.indicator.url.scheme -- external: ecs - name: threat.indicator.url.query -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.file.hash.sha512 -- external: ecs - name: threat.indicator.email.address -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.marking.tlp -- external: ecs - name: threat.indicator.confidence -- external: ecs - name: threat.indicator.as.number -- external: ecs - name: threat.indicator.as.organization.name -- external: ecs - name: threat.indicator.geo.location -- external: ecs - name: threat.indicator.geo.country_iso_code -- external: ecs - name: labels diff --git a/packages/ti_anomali/docs/README.md b/packages/ti_anomali/docs/README.md index c80c2e3c4ab..2c496c354c8 100644 --- a/packages/ti_anomali/docs/README.md +++ b/packages/ti_anomali/docs/README.md @@ -163,85 +163,19 @@ An example event for `threatstream` looks as following: | anomali.threatstream.update_id | Update ID. | keyword | | anomali.threatstream.url | URL for the indicator. | keyword | | anomali.threatstream.value_type | Data type of the indicator. Possible values: ip, domain, url, email, md5. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| labels | Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels. | object | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | | threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword | | threat.feed.name | Display friendly feed name | constant_keyword | -| threat.indicator.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| threat.indicator.as.organization.name | Organization name. | keyword | -| threat.indicator.as.organization.name.text | Multi-field of `threat.indicator.as.organization.name`. | match_only_text | -| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword | -| threat.indicator.email.address | Identifies a threat indicator as an email address (irrespective of direction). | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.file.hash.sha512 | SHA512 hash. | keyword | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.geo.country_iso_code | Country ISO code. | keyword | -| threat.indicator.geo.location | Longitude and latitude. | geo_point | -| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.marking.tlp | Traffic Light Protocol sharing markings. | keyword | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| threat.indicator.url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| threat.indicator.url.full.text | Multi-field of `threat.indicator.url.full`. | match_only_text | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | -| threat.indicator.url.path | Path of the request, such as "/search". | wildcard | -| threat.indicator.url.port | Port of the request, such as 443. | long | -| threat.indicator.url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| threat.indicator.url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | diff --git a/packages/ti_anomali/manifest.yml b/packages/ti_anomali/manifest.yml index f47b6d4b908..0a24809b304 100644 --- a/packages/ti_anomali/manifest.yml +++ b/packages/ti_anomali/manifest.yml @@ -1,13 +1,13 @@ name: ti_anomali title: Anomali -version: "1.21.0" +version: "1.22.0" description: Ingest threat intelligence indicators from Anomali with Elastic Agent. type: integration format_version: 3.0.2 categories: ["security", "threat_intel"] conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" icons: - src: /img/anomali.svg title: Anomali From 33d13c24213a5bc41246fbb4a012998b121b46f3 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:27:47 +0930 Subject: [PATCH 095/121] [ti_cif3] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/ti_cif3 --- packages/ti_cif3/changelog.yml | 5 + .../ti_cif3/data_stream/feed/fields/beats.yml | 3 - .../ti_cif3/data_stream/feed/fields/ecs.yml | 102 ------------------ packages/ti_cif3/docs/README.md | 55 ---------- packages/ti_cif3/manifest.yml | 4 +- 5 files changed, 7 insertions(+), 162 deletions(-) diff --git a/packages/ti_cif3/changelog.yml b/packages/ti_cif3/changelog.yml index 5d3177b7437..2fac90c48a9 100644 --- a/packages/ti_cif3/changelog.yml +++ b/packages/ti_cif3/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.14.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.13.1" changes: - description: Adjust field mappings for transform destination index. diff --git a/packages/ti_cif3/data_stream/feed/fields/beats.yml b/packages/ti_cif3/data_stream/feed/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/ti_cif3/data_stream/feed/fields/beats.yml +++ b/packages/ti_cif3/data_stream/feed/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/ti_cif3/data_stream/feed/fields/ecs.yml b/packages/ti_cif3/data_stream/feed/fields/ecs.yml index 18abb0a68db..5e8cd8465f7 100644 --- a/packages/ti_cif3/data_stream/feed/fields/ecs.yml +++ b/packages/ti_cif3/data_stream/feed/fields/ecs.yml @@ -1,106 +1,4 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: error.message -- external: ecs - name: tags -- external: ecs - name: related.hash -- external: ecs - name: related.ip -- external: ecs - name: related.hosts -- external: ecs - name: event.created -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.category -- external: ecs - name: event.provider -- external: ecs - name: event.type -- external: ecs - name: event.original -- external: ecs - name: network.protocol -- external: ecs - name: network.transport -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.first_seen -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.indicator.modified_at -- external: ecs - name: threat.indicator.reference -- external: ecs - name: threat.indicator.description -- external: ecs - name: threat.indicator.sightings -- external: ecs - name: threat.indicator.file.type -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.file.hash.sha512 -- external: ecs - name: threat.indicator.file.pe.imphash -- external: ecs - name: threat.indicator.file.hash.ssdeep - name: threat.indicator.tls.client.ja3 level: extended type: keyword description: An md5 hash that identifies clients based on their TLS handshake. -- external: ecs - name: threat.indicator.email.address -- external: ecs - name: threat.indicator.ip -- external: ecs - name: threat.indicator.url.domain -- external: ecs - name: threat.indicator.url.full -- external: ecs - name: threat.indicator.url.extension -- external: ecs - name: threat.indicator.url.original -- external: ecs - name: threat.indicator.url.path -- external: ecs - name: threat.indicator.url.port -- external: ecs - name: threat.indicator.url.scheme -- external: ecs - name: threat.indicator.url.query -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.as.number -- external: ecs - name: threat.indicator.as.organization.name -- external: ecs - name: threat.indicator.marking.tlp -- external: ecs - name: threat.indicator.confidence -- external: ecs - name: threat.indicator.geo.location -- external: ecs - name: threat.indicator.geo.country_iso_code -- external: ecs - name: threat.indicator.geo.region_name -- external: ecs - name: threat.indicator.geo.timezone -- external: ecs - name: threat.indicator.name -- external: ecs - name: labels diff --git a/packages/ti_cif3/docs/README.md b/packages/ti_cif3/docs/README.md index 666dae4a399..6dee6248430 100644 --- a/packages/ti_cif3/docs/README.md +++ b/packages/ti_cif3/docs/README.md @@ -72,69 +72,14 @@ CIFv3 `confidence` field values (0..10) are converted to ECS confidence (None, L | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Name of the module this data is coming from. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | input.type | Type of Filebeat input. | keyword | -| labels | Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels. | object | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| tags | List of keywords used to tag each event. | keyword | | threat.feed.name | Display friendly feed name | constant_keyword | -| threat.indicator.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| threat.indicator.as.organization.name | Organization name. | keyword | -| threat.indicator.as.organization.name.text | Multi-field of `threat.indicator.as.organization.name`. | match_only_text | -| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword | -| threat.indicator.description | Describes the type of action conducted by the threat. | keyword | -| threat.indicator.email.address | Identifies a threat indicator as an email address (irrespective of direction). | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.file.hash.sha512 | SHA512 hash. | keyword | -| threat.indicator.file.hash.ssdeep | SSDEEP hash. | keyword | -| threat.indicator.file.pe.imphash | A hash of the imports in a PE file. An imphash -- or import hash -- can be used to fingerprint binaries even after recompilation or other code-level transformations have occurred, which would change more traditional hash values. Learn more at https://www.fireeye.com/blog/threat-research/2014/01/tracking-malware-import-hashing.html. | keyword | -| threat.indicator.file.type | File type (file, dir, or symlink). | keyword | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.geo.country_iso_code | Country ISO code. | keyword | -| threat.indicator.geo.location | Longitude and latitude. | geo_point | -| threat.indicator.geo.region_name | Region name. | keyword | -| threat.indicator.geo.timezone | The time zone of the location, such as IANA time zone name. | keyword | -| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.marking.tlp | Traffic Light Protocol sharing markings. | keyword | -| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | -| threat.indicator.name | The display name indicator in an UI friendly format URL, IP address, email address, registry key, port number, hash value, or other relevant name can serve as the display name. | keyword | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.reference | Reference URL linking to additional information about this indicator. | keyword | -| threat.indicator.sightings | Number of times this indicator was observed conducting threat activity. | long | | threat.indicator.tls.client.ja3 | An md5 hash that identifies clients based on their TLS handshake. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| threat.indicator.url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| threat.indicator.url.full.text | Multi-field of `threat.indicator.url.full`. | match_only_text | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | -| threat.indicator.url.path | Path of the request, such as "/search". | wildcard | -| threat.indicator.url.port | Port of the request, such as 443. | long | -| threat.indicator.url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| threat.indicator.url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | An example event for `feed` looks as following: diff --git a/packages/ti_cif3/manifest.yml b/packages/ti_cif3/manifest.yml index 3580b2b3cc2..d5a258a9f2d 100644 --- a/packages/ti_cif3/manifest.yml +++ b/packages/ti_cif3/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: ti_cif3 title: "Collective Intelligence Framework v3" -version: "1.13.1" +version: "1.14.0" description: "Ingest threat indicators from a Collective Intelligence Framework v3 instance with Elastic Agent." type: integration categories: @@ -9,7 +9,7 @@ categories: - threat_intel conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" icons: - src: /img/csg_logo_big.svg title: csirtgadgets logo From 60ef8aa3958c68d3818d99a9dddb3b145bae0e87 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:27:49 +0930 Subject: [PATCH 096/121] [ti_crowdstrike] - Updated fields definitions Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/ti_crowdstrike --- packages/ti_crowdstrike/changelog.yml | 5 +++++ .../intel/_dev/test/pipeline/test-common-config.yml | 1 - packages/ti_crowdstrike/data_stream/intel/fields/beats.yml | 3 --- packages/ti_crowdstrike/data_stream/ioc/fields/beats.yml | 3 --- packages/ti_crowdstrike/docs/README.md | 2 -- .../elasticsearch/transform/latest_intel/fields/ecs.yml | 2 +- .../elasticsearch/transform/latest_ioc/fields/ecs.yml | 2 +- packages/ti_crowdstrike/manifest.yml | 2 +- 8 files changed, 8 insertions(+), 12 deletions(-) diff --git a/packages/ti_crowdstrike/changelog.yml b/packages/ti_crowdstrike/changelog.yml index 641e659e0f5..5d6ee86fef8 100644 --- a/packages/ti_crowdstrike/changelog.yml +++ b/packages/ti_crowdstrike/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.1.0" + changes: + - description: Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.0.1" changes: - description: Adjust field mappings for transform destination index. diff --git a/packages/ti_crowdstrike/data_stream/intel/_dev/test/pipeline/test-common-config.yml b/packages/ti_crowdstrike/data_stream/intel/_dev/test/pipeline/test-common-config.yml index 36106b22efb..1f0a54d166d 100644 --- a/packages/ti_crowdstrike/data_stream/intel/_dev/test/pipeline/test-common-config.yml +++ b/packages/ti_crowdstrike/data_stream/intel/_dev/test/pipeline/test-common-config.yml @@ -2,7 +2,6 @@ fields: tags: - preserve_original_event - preserve_duplicate_custom_fields - dynamic_fields: # This can be removed after ES 8.14 is the minimum version. # Relates: https://github.com/elastic/elasticsearch/pull/105689 diff --git a/packages/ti_crowdstrike/data_stream/intel/fields/beats.yml b/packages/ti_crowdstrike/data_stream/intel/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/ti_crowdstrike/data_stream/intel/fields/beats.yml +++ b/packages/ti_crowdstrike/data_stream/intel/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/ti_crowdstrike/data_stream/ioc/fields/beats.yml b/packages/ti_crowdstrike/data_stream/ioc/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/ti_crowdstrike/data_stream/ioc/fields/beats.yml +++ b/packages/ti_crowdstrike/data_stream/ioc/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/ti_crowdstrike/docs/README.md b/packages/ti_crowdstrike/docs/README.md index 3852df59040..638365f0f04 100644 --- a/packages/ti_crowdstrike/docs/README.md +++ b/packages/ti_crowdstrike/docs/README.md @@ -266,7 +266,6 @@ An example event for `intel` looks as following: | input.type | Type of filebeat input. | keyword | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | | threat.feed.name | Display friendly feed name. | constant_keyword | | ti_crowdstrike.intel._marker | A special marker associated with the Intel Indicator. | keyword | | ti_crowdstrike.intel.actors | Information related to actors associated with the Intel Indicator. | keyword | @@ -425,7 +424,6 @@ An example event for `ioc` looks as following: | input.type | Type of filebeat input. | keyword | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | | threat.feed.name | Display friendly feed name. | constant_keyword | | ti_crowdstrike.ioc.action | Describes the action taken when the IOC is detected. | keyword | | ti_crowdstrike.ioc.applied_globally | Indicates whether the IOC is applied globally. | boolean | diff --git a/packages/ti_crowdstrike/elasticsearch/transform/latest_intel/fields/ecs.yml b/packages/ti_crowdstrike/elasticsearch/transform/latest_intel/fields/ecs.yml index 630d61d9b7f..5ff0db55f80 100644 --- a/packages/ti_crowdstrike/elasticsearch/transform/latest_intel/fields/ecs.yml +++ b/packages/ti_crowdstrike/elasticsearch/transform/latest_intel/fields/ecs.yml @@ -51,4 +51,4 @@ - name: threat.feed.name type: constant_keyword description: Display friendly feed name. - value: CrowdStrike Intel \ No newline at end of file + value: CrowdStrike Intel diff --git a/packages/ti_crowdstrike/elasticsearch/transform/latest_ioc/fields/ecs.yml b/packages/ti_crowdstrike/elasticsearch/transform/latest_ioc/fields/ecs.yml index 93108af9ebb..60de423993f 100644 --- a/packages/ti_crowdstrike/elasticsearch/transform/latest_ioc/fields/ecs.yml +++ b/packages/ti_crowdstrike/elasticsearch/transform/latest_ioc/fields/ecs.yml @@ -55,4 +55,4 @@ - name: threat.feed.name type: constant_keyword description: Display friendly feed name. - value: CrowdStrike IOC \ No newline at end of file + value: CrowdStrike IOC diff --git a/packages/ti_crowdstrike/manifest.yml b/packages/ti_crowdstrike/manifest.yml index 4e17c778467..fbda4e8a894 100644 --- a/packages/ti_crowdstrike/manifest.yml +++ b/packages/ti_crowdstrike/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.3 name: ti_crowdstrike title: CrowdStrike Falcon Intelligence -version: 1.0.1 +version: "1.1.0" description: Collect logs from CrowdStrike Falcon Intelligence with Elastic Agent. type: integration categories: From fa4ac207f6ca4bef0974790f9bb0eca1c64d9268 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:27:50 +0930 Subject: [PATCH 097/121] [ti_cybersixgill] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/ti_cybersixgill --- packages/ti_cybersixgill/changelog.yml | 5 + .../data_stream/threat/fields/agent.yml | 167 +----------------- .../data_stream/threat/fields/ecs.yml | 69 -------- packages/ti_cybersixgill/docs/README.md | 61 ------- .../transform/latest_ioc/fields/ecs.yml | 2 +- packages/ti_cybersixgill/manifest.yml | 4 +- 6 files changed, 9 insertions(+), 299 deletions(-) delete mode 100644 packages/ti_cybersixgill/data_stream/threat/fields/ecs.yml diff --git a/packages/ti_cybersixgill/changelog.yml b/packages/ti_cybersixgill/changelog.yml index 5ed502d8ca7..2947ecc9b5a 100644 --- a/packages/ti_cybersixgill/changelog.yml +++ b/packages/ti_cybersixgill/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.30.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.29.1" changes: - description: Fix sample event. diff --git a/packages/ti_cybersixgill/data_stream/threat/fields/agent.yml b/packages/ti_cybersixgill/data_stream/threat/fields/agent.yml index 845b84ed9c0..4bdd88d3cd7 100644 --- a/packages/ti_cybersixgill/data_stream/threat/fields/agent.yml +++ b/packages/ti_cybersixgill/data_stream/threat/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/ti_cybersixgill/data_stream/threat/fields/ecs.yml b/packages/ti_cybersixgill/data_stream/threat/fields/ecs.yml deleted file mode 100644 index 022139ab678..00000000000 --- a/packages/ti_cybersixgill/data_stream/threat/fields/ecs.yml +++ /dev/null @@ -1,69 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: error.message -- external: ecs - name: event.category -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.severity -- external: ecs - name: event.created -- external: ecs - name: tags -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.url.full -- external: ecs - name: threat.indicator.url.domain -- external: ecs - name: threat.indicator.url.extension -- external: ecs - name: threat.indicator.url.original -- external: ecs - name: threat.indicator.url.path -- external: ecs - name: threat.indicator.url.scheme -- external: ecs - name: threat.indicator.url.fragment -- external: ecs - name: threat.indicator.ip -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.description -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.reference -- external: ecs - name: threat.indicator.confidence -- external: ecs - name: threat.indicator.first_seen -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.tactic.name -- external: ecs - name: threat.tactic.id -- external: ecs - name: threat.tactic.reference -# Manually define this as a workaround for failing tests and validation -- name: threat.indicator.name - level: extended - type: keyword - description: The display name indicator in an UI friendly format -- external: ecs - name: labels diff --git a/packages/ti_cybersixgill/docs/README.md b/packages/ti_cybersixgill/docs/README.md index 726fbde7b26..ef1b83205a0 100644 --- a/packages/ti_cybersixgill/docs/README.md +++ b/packages/ti_cybersixgill/docs/README.md @@ -19,19 +19,7 @@ To facilitate IOC expiration, source datastream-backed indices `.ds-logs-ti_cybe | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | cybersixgill.actor | The related actor for the indicator. | keyword | | cybersixgill.deleted_at | The timestamp when indicator is (or will be) expired. | date | | cybersixgill.expiration_duration | The configured expiration duration. | keyword | @@ -47,64 +35,15 @@ To facilitate IOC expiration, source datastream-backed indices `.ds-logs-ti_cybe | dataset.name | Dataset name. | constant_keyword | | dataset.namespace | Dataset namespace. | constant_keyword | | dataset.type | Dataset type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type. | keyword | -| labels | Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels. | object | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | | threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword | | threat.feed.name | Display friendly feed name | constant_keyword | -| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword | -| threat.indicator.description | Describes the type of action conducted by the threat. | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.name | The display name indicator in an UI friendly format | keyword | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.reference | Reference URL linking to additional information about this indicator. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| threat.indicator.url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | -| threat.indicator.url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| threat.indicator.url.full.text | Multi-field of `threat.indicator.url.full`. | match_only_text | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | -| threat.indicator.url.path | Path of the request, such as "/search". | wildcard | -| threat.indicator.url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| threat.tactic.id | The id of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/ ) | keyword | -| threat.tactic.name | Name of the type of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/) | keyword | -| threat.tactic.reference | The reference url of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/ ) | keyword | An example event for `threat` looks as following: diff --git a/packages/ti_cybersixgill/elasticsearch/transform/latest_ioc/fields/ecs.yml b/packages/ti_cybersixgill/elasticsearch/transform/latest_ioc/fields/ecs.yml index 9d58dae5eae..f38a6d7c1cb 100644 --- a/packages/ti_cybersixgill/elasticsearch/transform/latest_ioc/fields/ecs.yml +++ b/packages/ti_cybersixgill/elasticsearch/transform/latest_ioc/fields/ecs.yml @@ -82,4 +82,4 @@ - name: threat.feed.dashboard_id type: constant_keyword description: Dashboard ID used for Kibana CTI UI - value: ti_cybersixgill-c75353f0-5be8-11ec-9302-152fd766c738 \ No newline at end of file + value: ti_cybersixgill-c75353f0-5be8-11ec-9302-152fd766c738 diff --git a/packages/ti_cybersixgill/manifest.yml b/packages/ti_cybersixgill/manifest.yml index d22b1497ace..14458999d3c 100644 --- a/packages/ti_cybersixgill/manifest.yml +++ b/packages/ti_cybersixgill/manifest.yml @@ -1,13 +1,13 @@ name: ti_cybersixgill title: Cybersixgill -version: "1.29.1" +version: "1.30.0" description: Ingest threat intelligence indicators from Cybersixgill with Elastic Agent. type: integration format_version: "3.0.2" categories: ["security", "threat_intel"] conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" policy_templates: - name: cybersixgill title: Cybersixgill Threat Intel From 8aa2bc4b041780006b05dc6f022b098b320e8e03 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:27:51 +0930 Subject: [PATCH 098/121] [ti_eclecticiq] - change to ECS version git@v8.11.0 ECS version in build manifest changed from git@v8.10.0 to git@v8.11.0. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. The set ecs.version processor in pipelines was changed 8.11.0. Previously the pipeline was setting version 8.10.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. The ecs.version in sample_event.json files was changed to 8.11.0. Previously sample_event.json files contained 8.10.0. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/ti_eclecticiq --- packages/ti_eclecticiq/_dev/build/build.yml | 2 +- packages/ti_eclecticiq/changelog.yml | 5 + ...est-outgoing-feed-event.json-expected.json | 54 +++++------ .../elasticsearch/ingest_pipeline/default.yml | 2 +- .../data_stream/threat/fields/ecs.yml | 96 ------------------- .../data_stream/threat/sample_event.json | 4 +- packages/ti_eclecticiq/docs/README.md | 58 +---------- .../transform/latest_ioc/fields/ecs.yml | 2 +- packages/ti_eclecticiq/manifest.yml | 4 +- 9 files changed, 41 insertions(+), 186 deletions(-) delete mode 100644 packages/ti_eclecticiq/data_stream/threat/fields/ecs.yml diff --git a/packages/ti_eclecticiq/_dev/build/build.yml b/packages/ti_eclecticiq/_dev/build/build.yml index 49e8fdaa97d..2bfcfc223b0 100644 --- a/packages/ti_eclecticiq/_dev/build/build.yml +++ b/packages/ti_eclecticiq/_dev/build/build.yml @@ -1,3 +1,3 @@ dependencies: ecs: - reference: git@v8.10.0 + reference: "git@v8.11.0" diff --git a/packages/ti_eclecticiq/changelog.yml b/packages/ti_eclecticiq/changelog.yml index 66a87c88368..0539ee50b5b 100644 --- a/packages/ti_eclecticiq/changelog.yml +++ b/packages/ti_eclecticiq/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.1.0" + changes: + - description: ECS version updated to 8.11.0. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.0.1" changes: - description: Adjust field mappings for transform destination index. diff --git a/packages/ti_eclecticiq/data_stream/threat/_dev/test/pipeline/test-outgoing-feed-event.json-expected.json b/packages/ti_eclecticiq/data_stream/threat/_dev/test/pipeline/test-outgoing-feed-event.json-expected.json index cf21503aca1..8b39b3bc8e9 100644 --- a/packages/ti_eclecticiq/data_stream/threat/_dev/test/pipeline/test-outgoing-feed-event.json-expected.json +++ b/packages/ti_eclecticiq/data_stream/threat/_dev/test/pipeline/test-outgoing-feed-event.json-expected.json @@ -9,7 +9,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -38,7 +38,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -78,7 +78,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -127,7 +127,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -178,7 +178,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -229,7 +229,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -277,7 +277,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -329,7 +329,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -377,7 +377,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -427,7 +427,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -471,7 +471,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -503,7 +503,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -546,7 +546,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -592,7 +592,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -635,7 +635,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -678,7 +678,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -718,7 +718,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -761,7 +761,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -804,7 +804,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -847,7 +847,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -890,7 +890,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -937,7 +937,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -985,7 +985,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -1023,7 +1023,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -1056,7 +1056,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ @@ -1089,7 +1089,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "email": { "subject": "Test email subject" @@ -1122,7 +1122,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ diff --git a/packages/ti_eclecticiq/data_stream/threat/elasticsearch/ingest_pipeline/default.yml b/packages/ti_eclecticiq/data_stream/threat/elasticsearch/ingest_pipeline/default.yml index 0fd46ba1c30..6f820915d43 100644 --- a/packages/ti_eclecticiq/data_stream/threat/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_eclecticiq/data_stream/threat/elasticsearch/ingest_pipeline/default.yml @@ -4,7 +4,7 @@ processors: # hard coded fields - set: field: ecs.version - value: 8.10.0 + value: 8.11.0 - set: field: event.kind value: enrichment diff --git a/packages/ti_eclecticiq/data_stream/threat/fields/ecs.yml b/packages/ti_eclecticiq/data_stream/threat/fields/ecs.yml deleted file mode 100644 index e76a9ef82ea..00000000000 --- a/packages/ti_eclecticiq/data_stream/threat/fields/ecs.yml +++ /dev/null @@ -1,96 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: tags -- external: ecs - name: event.url -- external: ecs - name: event.created -- external: ecs - name: event.original -- external: ecs - name: event.start -- external: ecs - name: event.end -- external: ecs - name: event.provider -- external: ecs - name: event.category -- external: ecs - name: event.kind -- external: ecs - name: event.type -- external: ecs - name: threat.indicator.first_seen -- external: ecs - name: threat.indicator.confidence -- external: ecs - name: threat.indicator.marking.tlp -- name: threat.indicator.name - level: extended - type: keyword - description: The display name indicator in an UI friendly format -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.as.number -- external: ecs - name: vulnerability.id -- external: ecs - name: threat.indicator.url.domain -- external: ecs - name: threat.indicator.email.address -- external: ecs - name: email.subject -- external: ecs - name: threat.indicator.file.path -- external: ecs - name: threat.indicator.file.size -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.file.hash.sha384 -- external: ecs - name: threat.indicator.file.hash.sha512 -- external: ecs - name: threat.indicator.file.hash.ssdeep -- external: ecs - name: host.hostname -- external: ecs - name: threat.indicator.ip -- external: ecs - name: threat.software.name -- external: ecs - name: threat.software.type -- external: ecs - name: organization.name -- external: ecs - name: threat.indicator.url.port -- external: ecs - name: process.command_line -- external: ecs - name: process.name -- external: ecs - name: threat.indicator.url.full -- external: ecs - name: user_agent.original -- external: ecs - name: threat.indicator.registry.value -- external: ecs - name: rule.name -- external: ecs - name: threat.indicator.x509.serial_number -- external: ecs - name: server.mac -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip diff --git a/packages/ti_eclecticiq/data_stream/threat/sample_event.json b/packages/ti_eclecticiq/data_stream/threat/sample_event.json index 5615bf1147d..0f530dde506 100644 --- a/packages/ti_eclecticiq/data_stream/threat/sample_event.json +++ b/packages/ti_eclecticiq/data_stream/threat/sample_event.json @@ -6,14 +6,14 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ "threat" ], - "dataset": "ti_eclecticiq.threat", "created": "2023-06-08T12:00:30.187Z", + "dataset": "ti_eclecticiq.threat", "id": "XugasX/Bvu/150lNyQjzIGR0zZ8=", "kind": "enrichment", "original": "{\"calculated.relevancy\": \"0.68\", \"calculated.source_reliability\": \"A\", \"calculated.tlp\": \"GREEN\", \"diff\": \"add\", \"entity.id\": \"5e814485-012d-423d-b769-026bfed0f451\", \"entity.title\": \"Example\", \"entity.type\": \"malware\", \"meta.classification\": \"\", \"meta.confidence\": \"\", \"meta.entity_url\": \"https://test.com/entity/5e814485-012d-423d-b769-026bfed0f451\", \"meta.estimated_observed_time\": \"2019-07-09T17:42:44.777000+00:00\", \"meta.estimated_threat_end_time\": \"\", \"meta.estimated_threat_start_time\": \"2022-05-11T14:00:00.188000+00:00\", \"meta.ingest_time\": \"2023-06-08T12:00:30.187097+00:00\", \"meta.relevancy\": \"0.68\", \"meta.source_reliability\": \"A\", \"meta.tags\": \"tag1;tag2\", \"meta.taxonomy\": \"\", \"meta.terms_of_use\": \"\", \"meta.tlp\": \"GREEN\", \"source.ids\": \"47ec245c-9e7b-467e-a016-77a22ff12dd5\", \"source.names\": \"Test Source\", \"timestamp\": \"2023-06-20 18:06:10.126780+00:00\", \"type\": \"domain\", \"value\": \"example.com\", \"value_url\": \"https://test.com/main/extracts/domain/test\"}", diff --git a/packages/ti_eclecticiq/docs/README.md b/packages/ti_eclecticiq/docs/README.md index c4a3c23fc84..e49747ffd0e 100644 --- a/packages/ti_eclecticiq/docs/README.md +++ b/packages/ti_eclecticiq/docs/README.md @@ -186,14 +186,14 @@ An example event for `threat` looks as following: } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "category": [ "threat" ], - "dataset": "ti_eclecticiq.threat", "created": "2023-06-08T12:00:30.187Z", + "dataset": "ti_eclecticiq.threat", "id": "XugasX/Bvu/150lNyQjzIGR0zZ8=", "kind": "enrichment", "original": "{\"calculated.relevancy\": \"0.68\", \"calculated.source_reliability\": \"A\", \"calculated.tlp\": \"GREEN\", \"diff\": \"add\", \"entity.id\": \"5e814485-012d-423d-b769-026bfed0f451\", \"entity.title\": \"Example\", \"entity.type\": \"malware\", \"meta.classification\": \"\", \"meta.confidence\": \"\", \"meta.entity_url\": \"https://test.com/entity/5e814485-012d-423d-b769-026bfed0f451\", \"meta.estimated_observed_time\": \"2019-07-09T17:42:44.777000+00:00\", \"meta.estimated_threat_end_time\": \"\", \"meta.estimated_threat_start_time\": \"2022-05-11T14:00:00.188000+00:00\", \"meta.ingest_time\": \"2023-06-08T12:00:30.187097+00:00\", \"meta.relevancy\": \"0.68\", \"meta.source_reliability\": \"A\", \"meta.tags\": \"tag1;tag2\", \"meta.taxonomy\": \"\", \"meta.terms_of_use\": \"\", \"meta.tlp\": \"GREEN\", \"source.ids\": \"47ec245c-9e7b-467e-a016-77a22ff12dd5\", \"source.names\": \"Test Source\", \"timestamp\": \"2023-06-20 18:06:10.126780+00:00\", \"type\": \"domain\", \"value\": \"example.com\", \"value_url\": \"https://test.com/main/extracts/domain/test\"}", @@ -236,63 +236,9 @@ An example event for `threat` looks as following: | data_stream.type | Data stream type. | constant_keyword | | eclecticiq.threat.deleted_at | Date when observable was removed from dataset | date | | eclecticiq.threat.observable_id | The ID of the observable, based on kind and value. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| email.subject | A brief summary of the topic of the message. | keyword | -| email.subject.text | Multi-field of `email.subject`. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| event.url | URL linking to an external system to continue investigation of this event. This URL links to another system where in-depth investigation of the specific occurrence of this event can take place. Alert events, indicated by `event.kind:alert`, are a common use case for this field. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | input.type | Input type | keyword | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| organization.name | Organization name. | keyword | -| organization.name.text | Multi-field of `organization.name`. | match_only_text | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| rule.name | The name of the rule or signature generating the event. | keyword | -| server.mac | MAC address of the server. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| tags | List of keywords used to tag each event. | keyword | | threat.feed.name | Display friendly feed name | constant_keyword | -| threat.indicator.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword | -| threat.indicator.email.address | Identifies a threat indicator as an email address (irrespective of direction). | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.file.hash.sha384 | SHA384 hash. | keyword | -| threat.indicator.file.hash.sha512 | SHA512 hash. | keyword | -| threat.indicator.file.hash.ssdeep | SSDEEP hash. | keyword | -| threat.indicator.file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| threat.indicator.file.path.text | Multi-field of `threat.indicator.file.path`. | match_only_text | -| threat.indicator.file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | -| threat.indicator.marking.tlp | Traffic Light Protocol sharing markings. | keyword | -| threat.indicator.name | The display name indicator in an UI friendly format | keyword | -| threat.indicator.registry.value | Name of the value written. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| threat.indicator.url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| threat.indicator.url.full.text | Multi-field of `threat.indicator.url.full`. | match_only_text | -| threat.indicator.url.port | Port of the request, such as 443. | long | -| threat.indicator.x509.serial_number | Unique serial number issued by the certificate authority. For consistency, if this value is alphanumeric, it should be formatted without colons and uppercase characters. | keyword | -| threat.software.name | The name of the software used by this threat to conduct behavior commonly modeled using MITRE ATT&CK®. While not required, you can use a MITRE ATT&CK® software name. | keyword | -| threat.software.type | The type of software used by this threat to conduct behavior commonly modeled using MITRE ATT&CK®. While not required, you can use a MITRE ATT&CK® software type. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| vulnerability.id | The identification (ID) is the number portion of a vulnerability entry. It includes a unique identification number for the vulnerability. For example (https://cve.mitre.org/about/faqs.html#what_is_cve_id)[Common Vulnerabilities and Exposure CVE ID] | keyword | diff --git a/packages/ti_eclecticiq/elasticsearch/transform/latest_ioc/fields/ecs.yml b/packages/ti_eclecticiq/elasticsearch/transform/latest_ioc/fields/ecs.yml index 755244bdb14..62a39367662 100644 --- a/packages/ti_eclecticiq/elasticsearch/transform/latest_ioc/fields/ecs.yml +++ b/packages/ti_eclecticiq/elasticsearch/transform/latest_ioc/fields/ecs.yml @@ -107,4 +107,4 @@ - name: threat.feed.name type: constant_keyword description: Display friendly feed name - value: EclecticIQ \ No newline at end of file + value: EclecticIQ diff --git a/packages/ti_eclecticiq/manifest.yml b/packages/ti_eclecticiq/manifest.yml index 6ede1f0d9e9..0c1933fa3b2 100644 --- a/packages/ti_eclecticiq/manifest.yml +++ b/packages/ti_eclecticiq/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.3 name: ti_eclecticiq title: EclecticIQ -version: 1.0.1 +version: "1.1.0" description: Ingest threat intelligence from EclecticIQ with Elastic Agent type: integration categories: @@ -9,7 +9,7 @@ categories: - threat_intel conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: subscription: basic icons: From 81e93de7b5708d0ef404e032fb383369f76037ac Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:27:57 +0930 Subject: [PATCH 099/121] [ti_eset] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/ti_eset --- packages/ti_eset/changelog.yml | 5 + .../ti_eset/data_stream/apt/fields/agent.yml | 167 +-------- .../ti_eset/data_stream/apt/fields/ecs.yml | 92 ----- .../data_stream/botnet/fields/agent.yml | 167 +-------- .../ti_eset/data_stream/botnet/fields/ecs.yml | 42 --- .../ti_eset/data_stream/cc/fields/agent.yml | 167 +-------- .../ti_eset/data_stream/cc/fields/ecs.yml | 36 -- .../data_stream/domains/fields/agent.yml | 167 +-------- .../data_stream/domains/fields/ecs.yml | 38 -- .../data_stream/files/fields/agent.yml | 167 +-------- .../ti_eset/data_stream/files/fields/ecs.yml | 42 --- .../ti_eset/data_stream/ip/fields/agent.yml | 167 +-------- .../ti_eset/data_stream/ip/fields/ecs.yml | 40 -- .../ti_eset/data_stream/url/fields/agent.yml | 167 +-------- .../ti_eset/data_stream/url/fields/ecs.yml | 36 -- packages/ti_eset/docs/README.md | 352 ------------------ packages/ti_eset/manifest.yml | 4 +- 17 files changed, 14 insertions(+), 1842 deletions(-) delete mode 100644 packages/ti_eset/data_stream/apt/fields/ecs.yml delete mode 100644 packages/ti_eset/data_stream/botnet/fields/ecs.yml delete mode 100644 packages/ti_eset/data_stream/cc/fields/ecs.yml delete mode 100644 packages/ti_eset/data_stream/domains/fields/ecs.yml delete mode 100644 packages/ti_eset/data_stream/files/fields/ecs.yml delete mode 100644 packages/ti_eset/data_stream/ip/fields/ecs.yml delete mode 100644 packages/ti_eset/data_stream/url/fields/ecs.yml diff --git a/packages/ti_eset/changelog.yml b/packages/ti_eset/changelog.yml index 3bced928f0a..d7b6ea60178 100644 --- a/packages/ti_eset/changelog.yml +++ b/packages/ti_eset/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.2.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.1.1" changes: - description: Adjust field mappings for transform destination index. diff --git a/packages/ti_eset/data_stream/apt/fields/agent.yml b/packages/ti_eset/data_stream/apt/fields/agent.yml index 845b84ed9c0..4bdd88d3cd7 100644 --- a/packages/ti_eset/data_stream/apt/fields/agent.yml +++ b/packages/ti_eset/data_stream/apt/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/ti_eset/data_stream/apt/fields/ecs.yml b/packages/ti_eset/data_stream/apt/fields/ecs.yml deleted file mode 100644 index 5162c5f7184..00000000000 --- a/packages/ti_eset/data_stream/apt/fields/ecs.yml +++ /dev/null @@ -1,92 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: error.message -- external: ecs - name: event.category -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.created -- external: ecs - name: tags -- external: ecs - name: threat.indicator.name -- external: ecs - name: threat.indicator.confidence -- external: ecs - name: threat.indicator.url.original -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.description -- external: ecs - name: threat.indicator.modified_at -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.feed.name -- external: ecs - name: threat.indicator.email.address -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.file.name -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.url.domain -- external: ecs - name: threat.indicator.url.path -- external: ecs - name: threat.indicator.url.port -- external: ecs - name: threat.indicator.url.scheme -- external: ecs - name: threat.indicator.x509.issuer.common_name -- external: ecs - name: threat.indicator.x509.issuer.country -- external: ecs - name: threat.indicator.x509.issuer.distinguished_name -- external: ecs - name: threat.indicator.x509.issuer.locality -- external: ecs - name: threat.indicator.x509.issuer.organization -- external: ecs - name: threat.indicator.x509.issuer.state_or_province -- external: ecs - name: threat.indicator.x509.issuer.organizational_unit -- external: ecs - name: threat.indicator.x509.not_after -- external: ecs - name: threat.indicator.x509.not_before -- external: ecs - name: threat.indicator.x509.serial_number -- external: ecs - name: threat.indicator.x509.signature_algorithm -- external: ecs - name: threat.indicator.x509.subject.common_name -- external: ecs - name: threat.indicator.x509.subject.country -- external: ecs - name: threat.indicator.x509.subject.distinguished_name -- external: ecs - name: threat.indicator.x509.subject.locality -- external: ecs - name: threat.indicator.x509.subject.organization -- external: ecs - name: threat.indicator.x509.subject.state_or_province -- external: ecs - name: threat.indicator.x509.subject.organizational_unit -- external: ecs - name: threat.indicator.x509.version_number diff --git a/packages/ti_eset/data_stream/botnet/fields/agent.yml b/packages/ti_eset/data_stream/botnet/fields/agent.yml index 845b84ed9c0..4bdd88d3cd7 100644 --- a/packages/ti_eset/data_stream/botnet/fields/agent.yml +++ b/packages/ti_eset/data_stream/botnet/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/ti_eset/data_stream/botnet/fields/ecs.yml b/packages/ti_eset/data_stream/botnet/fields/ecs.yml deleted file mode 100644 index 43534883f1c..00000000000 --- a/packages/ti_eset/data_stream/botnet/fields/ecs.yml +++ /dev/null @@ -1,42 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: error.message -- external: ecs - name: event.category -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.created -- external: ecs - name: tags -- external: ecs - name: threat.indicator.name -- external: ecs - name: threat.indicator.confidence -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.url.original -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.description -- external: ecs - name: threat.indicator.modified_at -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.feed.name diff --git a/packages/ti_eset/data_stream/cc/fields/agent.yml b/packages/ti_eset/data_stream/cc/fields/agent.yml index 845b84ed9c0..4bdd88d3cd7 100644 --- a/packages/ti_eset/data_stream/cc/fields/agent.yml +++ b/packages/ti_eset/data_stream/cc/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/ti_eset/data_stream/cc/fields/ecs.yml b/packages/ti_eset/data_stream/cc/fields/ecs.yml deleted file mode 100644 index d3f9633c4c9..00000000000 --- a/packages/ti_eset/data_stream/cc/fields/ecs.yml +++ /dev/null @@ -1,36 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: error.message -- external: ecs - name: event.category -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.created -- external: ecs - name: tags -- external: ecs - name: threat.indicator.name -- external: ecs - name: threat.indicator.confidence -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.url.original -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.description -- external: ecs - name: threat.indicator.modified_at -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.feed.name diff --git a/packages/ti_eset/data_stream/domains/fields/agent.yml b/packages/ti_eset/data_stream/domains/fields/agent.yml index 845b84ed9c0..4bdd88d3cd7 100644 --- a/packages/ti_eset/data_stream/domains/fields/agent.yml +++ b/packages/ti_eset/data_stream/domains/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/ti_eset/data_stream/domains/fields/ecs.yml b/packages/ti_eset/data_stream/domains/fields/ecs.yml deleted file mode 100644 index f127a34e100..00000000000 --- a/packages/ti_eset/data_stream/domains/fields/ecs.yml +++ /dev/null @@ -1,38 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: error.message -- external: ecs - name: event.category -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.created -- external: ecs - name: tags -- external: ecs - name: threat.indicator.name -- external: ecs - name: threat.indicator.confidence -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.url.original -- external: ecs - name: threat.indicator.url.domain -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.description -- external: ecs - name: threat.indicator.modified_at -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.feed.name diff --git a/packages/ti_eset/data_stream/files/fields/agent.yml b/packages/ti_eset/data_stream/files/fields/agent.yml index 845b84ed9c0..4bdd88d3cd7 100644 --- a/packages/ti_eset/data_stream/files/fields/agent.yml +++ b/packages/ti_eset/data_stream/files/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/ti_eset/data_stream/files/fields/ecs.yml b/packages/ti_eset/data_stream/files/fields/ecs.yml deleted file mode 100644 index 43534883f1c..00000000000 --- a/packages/ti_eset/data_stream/files/fields/ecs.yml +++ /dev/null @@ -1,42 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: error.message -- external: ecs - name: event.category -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.created -- external: ecs - name: tags -- external: ecs - name: threat.indicator.name -- external: ecs - name: threat.indicator.confidence -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.url.original -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.description -- external: ecs - name: threat.indicator.modified_at -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.feed.name diff --git a/packages/ti_eset/data_stream/ip/fields/agent.yml b/packages/ti_eset/data_stream/ip/fields/agent.yml index 845b84ed9c0..4bdd88d3cd7 100644 --- a/packages/ti_eset/data_stream/ip/fields/agent.yml +++ b/packages/ti_eset/data_stream/ip/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/ti_eset/data_stream/ip/fields/ecs.yml b/packages/ti_eset/data_stream/ip/fields/ecs.yml deleted file mode 100644 index 532e63297da..00000000000 --- a/packages/ti_eset/data_stream/ip/fields/ecs.yml +++ /dev/null @@ -1,40 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: error.message -- external: ecs - name: event.category -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.created -- external: ecs - name: tags -- external: ecs - name: threat.indicator.name -- external: ecs - name: threat.indicator.confidence -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.url.original -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.description -- external: ecs - name: threat.indicator.modified_at -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.feed.name -- external: ecs - name: threat.indicator.ip -- external: ecs - name: threat.indicator.port diff --git a/packages/ti_eset/data_stream/url/fields/agent.yml b/packages/ti_eset/data_stream/url/fields/agent.yml index 845b84ed9c0..4bdd88d3cd7 100644 --- a/packages/ti_eset/data_stream/url/fields/agent.yml +++ b/packages/ti_eset/data_stream/url/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/ti_eset/data_stream/url/fields/ecs.yml b/packages/ti_eset/data_stream/url/fields/ecs.yml deleted file mode 100644 index d3f9633c4c9..00000000000 --- a/packages/ti_eset/data_stream/url/fields/ecs.yml +++ /dev/null @@ -1,36 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: error.message -- external: ecs - name: event.category -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.created -- external: ecs - name: tags -- external: ecs - name: threat.indicator.name -- external: ecs - name: threat.indicator.confidence -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.url.original -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.description -- external: ecs - name: threat.indicator.modified_at -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.feed.name diff --git a/packages/ti_eset/docs/README.md b/packages/ti_eset/docs/README.md index c24edb7da7f..58315ad5f23 100644 --- a/packages/ti_eset/docs/README.md +++ b/packages/ti_eset/docs/README.md @@ -97,68 +97,20 @@ The minimum **Kibana version** required is **8.12.0**. | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | | eset.id | The UID of the event object. | keyword | | eset.labels | Threat labels. | keyword | | eset.valid_until | Event expiration date. | date | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type. | keyword | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| threat.feed.name | The name of the threat feed in UI friendly format. | keyword | -| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword | -| threat.indicator.description | Describes the type of action conducted by the threat. | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | -| threat.indicator.name | The display name indicator in an UI friendly format URL, IP address, email address, registry key, port number, hash value, or other relevant name can serve as the display name. | keyword | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | An example event for `botnet` looks as following: @@ -246,65 +198,20 @@ An example event for `botnet` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | | eset.id | The UID of the event object. | keyword | | eset.labels | Threat labels. | keyword | | eset.valid_until | Event expiration date. | date | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type. | keyword | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| threat.feed.name | The name of the threat feed in UI friendly format. | keyword | -| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword | -| threat.indicator.description | Describes the type of action conducted by the threat. | keyword | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | -| threat.indicator.name | The display name indicator in an UI friendly format URL, IP address, email address, registry key, port number, hash value, or other relevant name can serve as the display name. | keyword | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | An example event for `cc` looks as following: @@ -388,66 +295,20 @@ An example event for `cc` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | | eset.id | The UID of the event object. | keyword | | eset.labels | Threat labels. | keyword | | eset.valid_until | Event expiration date. | date | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type. | keyword | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| threat.feed.name | The name of the threat feed in UI friendly format. | keyword | -| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword | -| threat.indicator.description | Describes the type of action conducted by the threat. | keyword | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | -| threat.indicator.name | The display name indicator in an UI friendly format URL, IP address, email address, registry key, port number, hash value, or other relevant name can serve as the display name. | keyword | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | An example event for `domains` looks as following: @@ -532,68 +393,20 @@ An example event for `domains` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | | eset.id | The UID of the event object. | keyword | | eset.labels | Threat labels. | keyword | | eset.valid_until | Event expiration date. | date | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type. | keyword | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| threat.feed.name | The name of the threat feed in UI friendly format. | keyword | -| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword | -| threat.indicator.description | Describes the type of action conducted by the threat. | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | -| threat.indicator.name | The display name indicator in an UI friendly format URL, IP address, email address, registry key, port number, hash value, or other relevant name can serve as the display name. | keyword | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | An example event for `files` looks as following: @@ -681,67 +494,20 @@ An example event for `files` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | | eset.id | The UID of the event object. | keyword | | eset.labels | Threat labels. | keyword | | eset.valid_until | Event expiration date. | date | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type. | keyword | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| threat.feed.name | The name of the threat feed in UI friendly format. | keyword | -| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword | -| threat.indicator.description | Describes the type of action conducted by the threat. | keyword | -| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | -| threat.indicator.name | The display name indicator in an UI friendly format URL, IP address, email address, registry key, port number, hash value, or other relevant name can serve as the display name. | keyword | -| threat.indicator.port | Identifies a threat indicator as a port number (irrespective of direction). | long | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | An example event for `ip` looks as following: @@ -823,96 +589,23 @@ An example event for `ip` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | | eset.category | Event category as defined by MISP. | keyword | | eset.id | The UID of the event object. | keyword | | eset.meta_category | Event sub-category as defined by MISP. | keyword | | eset.name | Human readable name describing the event. | keyword | | eset.type | Type of the event. | keyword | | eset.valid_until | Event expiration date. | date | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type. | keyword | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| threat.feed.name | The name of the threat feed in UI friendly format. | keyword | -| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword | -| threat.indicator.description | Describes the type of action conducted by the threat. | keyword | -| threat.indicator.email.address | Identifies a threat indicator as an email address (irrespective of direction). | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.file.name | Name of the file including the extension, without the directory. | keyword | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | -| threat.indicator.name | The display name indicator in an UI friendly format URL, IP address, email address, registry key, port number, hash value, or other relevant name can serve as the display name. | keyword | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | -| threat.indicator.url.path | Path of the request, such as "/search". | wildcard | -| threat.indicator.url.port | Port of the request, such as 443. | long | -| threat.indicator.url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| threat.indicator.x509.issuer.common_name | List of common name (CN) of issuing certificate authority. | keyword | -| threat.indicator.x509.issuer.country | List of country \(C) codes | keyword | -| threat.indicator.x509.issuer.distinguished_name | Distinguished name (DN) of issuing certificate authority. | keyword | -| threat.indicator.x509.issuer.locality | List of locality names (L) | keyword | -| threat.indicator.x509.issuer.organization | List of organizations (O) of issuing certificate authority. | keyword | -| threat.indicator.x509.issuer.organizational_unit | List of organizational units (OU) of issuing certificate authority. | keyword | -| threat.indicator.x509.issuer.state_or_province | List of state or province names (ST, S, or P) | keyword | -| threat.indicator.x509.not_after | Time at which the certificate is no longer considered valid. | date | -| threat.indicator.x509.not_before | Time at which the certificate is first considered valid. | date | -| threat.indicator.x509.serial_number | Unique serial number issued by the certificate authority. For consistency, if this value is alphanumeric, it should be formatted without colons and uppercase characters. | keyword | -| threat.indicator.x509.signature_algorithm | Identifier for certificate signature algorithm. We recommend using names found in Go Lang Crypto library. See https://github.com/golang/go/blob/go1.14/src/crypto/x509/x509.go#L337-L353. | keyword | -| threat.indicator.x509.subject.common_name | List of common names (CN) of subject. | keyword | -| threat.indicator.x509.subject.country | List of country \(C) code | keyword | -| threat.indicator.x509.subject.distinguished_name | Distinguished name (DN) of the certificate subject entity. | keyword | -| threat.indicator.x509.subject.locality | List of locality names (L) | keyword | -| threat.indicator.x509.subject.organization | List of organizations (O) of subject. | keyword | -| threat.indicator.x509.subject.organizational_unit | List of organizational units (OU) of subject. | keyword | -| threat.indicator.x509.subject.state_or_province | List of state or province names (ST, S, or P) | keyword | -| threat.indicator.x509.version_number | Version of x509 format. | keyword | An example event for `apt` looks as following: @@ -998,65 +691,20 @@ An example event for `apt` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | | eset.id | The UID of the event object. | keyword | | eset.labels | Threat labels. | keyword | | eset.valid_until | Event expiration date. | date | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type. | keyword | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| threat.feed.name | The name of the threat feed in UI friendly format. | keyword | -| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword | -| threat.indicator.description | Describes the type of action conducted by the threat. | keyword | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | -| threat.indicator.name | The display name indicator in an UI friendly format URL, IP address, email address, registry key, port number, hash value, or other relevant name can serve as the display name. | keyword | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | An example event for `url` looks as following: diff --git a/packages/ti_eset/manifest.yml b/packages/ti_eset/manifest.yml index 251198e8869..5d49543d3a5 100644 --- a/packages/ti_eset/manifest.yml +++ b/packages/ti_eset/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.3 name: ti_eset title: "ESET Threat Intelligence" -version: 1.1.1 +version: "1.2.0" description: "Ingest threat intelligence indicators from ESET Threat Intelligence with Elastic Agent." type: integration categories: @@ -9,7 +9,7 @@ categories: - threat_intel conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" elastic: subscription: "basic" screenshots: From e2cb6d9efe942e4d98eeacc38e64703ad02858d0 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:27:58 +0930 Subject: [PATCH 100/121] [ti_maltiverse] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/ti_maltiverse --- packages/ti_maltiverse/changelog.yml | 5 ++ .../data_stream/indicator/fields/ecs.yml | 88 ------------------- packages/ti_maltiverse/docs/README.md | 47 ---------- packages/ti_maltiverse/manifest.yml | 4 +- 4 files changed, 7 insertions(+), 137 deletions(-) delete mode 100644 packages/ti_maltiverse/data_stream/indicator/fields/ecs.yml diff --git a/packages/ti_maltiverse/changelog.yml b/packages/ti_maltiverse/changelog.yml index 859efb5d827..0f30d18f9ef 100644 --- a/packages/ti_maltiverse/changelog.yml +++ b/packages/ti_maltiverse/changelog.yml @@ -1,3 +1,8 @@ +- version: "1.2.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.1.1" changes: - description: Add missing fields for detection rules. diff --git a/packages/ti_maltiverse/data_stream/indicator/fields/ecs.yml b/packages/ti_maltiverse/data_stream/indicator/fields/ecs.yml deleted file mode 100644 index 92f7f496949..00000000000 --- a/packages/ti_maltiverse/data_stream/indicator/fields/ecs.yml +++ /dev/null @@ -1,88 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: tags -- external: ecs - name: error.message -- external: ecs - name: event.severity -- external: ecs - name: event.category -- external: ecs - name: event.id -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.type -- external: ecs - name: event.created -- external: ecs - name: event.original -- external: ecs - name: threat.indicator.description -- external: ecs - name: threat.feed.reference -- external: ecs - name: threat.indicator.first_seen -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.ip -- external: ecs - name: threat.indicator.url.domain -- external: ecs - name: threat.indicator.url.full -- external: ecs - name: threat.indicator.url.extension -- external: ecs - name: threat.indicator.url.original -- external: ecs - name: threat.indicator.url.path -- external: ecs - name: threat.indicator.url.port -- external: ecs - name: threat.indicator.url.scheme -- external: ecs - name: threat.indicator.url.query -- external: ecs - name: threat.indicator.url.registered_domain -- external: ecs - name: threat.indicator.url.top_level_domain -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.file.hash.sha512 -- external: ecs - name: threat.indicator.email.address -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.marking.tlp -- external: ecs - name: threat.indicator.confidence -- external: ecs - name: threat.indicator.as.number -- external: ecs - name: threat.indicator.as.organization.name -- external: ecs - name: threat.indicator.geo.location -- external: ecs - name: threat.indicator.geo.city_name -- external: ecs - name: threat.indicator.geo.country_iso_code -- external: ecs - name: threat.indicator.sightings -- external: ecs - name: threat.indicator.reference -- external: ecs - name: labels diff --git a/packages/ti_maltiverse/docs/README.md b/packages/ti_maltiverse/docs/README.md index 78bd973b5a3..b78c32f1c83 100644 --- a/packages/ti_maltiverse/docs/README.md +++ b/packages/ti_maltiverse/docs/README.md @@ -27,20 +27,9 @@ Both, the data_stream and the _latest index have applied expiration through ILM | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | input.type | Input type. | keyword | -| labels | Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels. | object | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | | maltiverse.address | registered address | keyword | | maltiverse.address.address | Multi-field of `maltiverse.address`. | match_only_text | @@ -100,42 +89,6 @@ Both, the data_stream and the _latest index have applied expiration through ILM | maltiverse.tag | Tags of the threat | keyword | | maltiverse.type | Type of the threat | keyword | | maltiverse.urlchecksum | | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | -| threat.feed.reference | Reference information for the threat feed in a UI friendly format. | keyword | -| threat.indicator.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| threat.indicator.as.organization.name | Organization name. | keyword | -| threat.indicator.as.organization.name.text | Multi-field of `threat.indicator.as.organization.name`. | match_only_text | -| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword | -| threat.indicator.description | Describes the type of action conducted by the threat. | keyword | -| threat.indicator.email.address | Identifies a threat indicator as an email address (irrespective of direction). | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.file.hash.sha512 | SHA512 hash. | keyword | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.geo.city_name | City name. | keyword | -| threat.indicator.geo.country_iso_code | Country ISO code. | keyword | -| threat.indicator.geo.location | Longitude and latitude. | geo_point | -| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.marking.tlp | Traffic Light Protocol sharing markings. | keyword | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.reference | Reference URL linking to additional information about this indicator. | keyword | -| threat.indicator.sightings | Number of times this indicator was observed conducting threat activity. | long | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| threat.indicator.url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| threat.indicator.url.full.text | Multi-field of `threat.indicator.url.full`. | match_only_text | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | -| threat.indicator.url.path | Path of the request, such as "/search". | wildcard | -| threat.indicator.url.port | Port of the request, such as 443. | long | -| threat.indicator.url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| threat.indicator.url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| threat.indicator.url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| threat.indicator.url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | An example event for `indicator` looks as following: diff --git a/packages/ti_maltiverse/manifest.yml b/packages/ti_maltiverse/manifest.yml index 08c6e9d5a37..f9fe5f2ba49 100644 --- a/packages/ti_maltiverse/manifest.yml +++ b/packages/ti_maltiverse/manifest.yml @@ -1,13 +1,13 @@ name: ti_maltiverse title: Maltiverse -version: "1.1.1" +version: "1.2.0" description: Ingest threat intelligence indicators from Maltiverse feeds with Elastic Agent type: integration format_version: 3.0.2 categories: ["security", "threat_intel"] conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" icons: - src: /img/logo-maltiverse.svg title: Maltiverse From 0573986c97f3e87242594a197a3d726857c16240 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:27:59 +0930 Subject: [PATCH 101/121] [ti_mandiant_advantage] - change to ECS version git@v8.11.0 ECS version in build manifest changed from git@8.11 to git@v8.11.0. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. The ecs.version in sample_event.json files was changed to 8.11.0. Previously sample_event.json files contained 8.7.0. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/ti_mandiant_advantage --- .../_dev/build/build.yml | 2 +- packages/ti_mandiant_advantage/changelog.yml | 5 + .../threat_intelligence/fields/ecs.yml | 161 ------------------ .../threat_intelligence/sample_event.json | 4 +- packages/ti_mandiant_advantage/docs/README.md | 87 +--------- packages/ti_mandiant_advantage/manifest.yml | 4 +- 6 files changed, 12 insertions(+), 251 deletions(-) diff --git a/packages/ti_mandiant_advantage/_dev/build/build.yml b/packages/ti_mandiant_advantage/_dev/build/build.yml index b33ec9554e4..2bfcfc223b0 100644 --- a/packages/ti_mandiant_advantage/_dev/build/build.yml +++ b/packages/ti_mandiant_advantage/_dev/build/build.yml @@ -1,3 +1,3 @@ dependencies: ecs: - reference: git@8.11 + reference: "git@v8.11.0" diff --git a/packages/ti_mandiant_advantage/changelog.yml b/packages/ti_mandiant_advantage/changelog.yml index c357cb02190..fc6f6a0b178 100644 --- a/packages/ti_mandiant_advantage/changelog.yml +++ b/packages/ti_mandiant_advantage/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.3.0" + changes: + - description: ECS version updated to 8.11.0. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.2.0" changes: - description: Improve handling of empty responses. diff --git a/packages/ti_mandiant_advantage/data_stream/threat_intelligence/fields/ecs.yml b/packages/ti_mandiant_advantage/data_stream/threat_intelligence/fields/ecs.yml index b99f4c3fe77..3c8e64b475d 100644 --- a/packages/ti_mandiant_advantage/data_stream/threat_intelligence/fields/ecs.yml +++ b/packages/ti_mandiant_advantage/data_stream/threat_intelligence/fields/ecs.yml @@ -1,164 +1,3 @@ - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider -- external: ecs - name: cloud.region -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.module -- external: ecs - name: event.original -- external: ecs - name: event.risk_score -- external: ecs - name: event.type -- external: ecs - name: related.hash -- external: ecs - name: related.ip -- external: ecs - name: tags -- external: ecs - name: threat.indicator.as.number -- external: ecs - name: threat.indicator.as.organization.name -- external: ecs - name: threat.indicator.confidence -- external: ecs - name: threat.indicator.description -- external: ecs - name: threat.indicator.email.address -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.file.hash.sha384 -- external: ecs - name: threat.indicator.file.hash.sha512 -- external: ecs - name: threat.indicator.first_seen -- external: ecs - name: threat.indicator.geo.city_name -- external: ecs - name: threat.indicator.geo.continent_name -- external: ecs - name: threat.indicator.geo.country_iso_code -- external: ecs - name: threat.indicator.geo.country_name -- external: ecs - name: threat.indicator.geo.location -- external: ecs - name: threat.indicator.geo.region_iso_code -- external: ecs - name: threat.indicator.geo.region_name -- external: ecs - name: threat.indicator.ip -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.indicator.modified_at -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.url.domain -- external: ecs - name: threat.indicator.url.extension -- external: ecs - name: threat.indicator.url.fragment -- external: ecs - name: threat.indicator.url.full - ignore_above: 4096 -- external: ecs - name: threat.indicator.url.original - ignore_above: 4096 -- external: ecs - name: threat.indicator.url.password -- external: ecs - name: threat.indicator.url.path -- external: ecs - name: threat.indicator.url.port -- external: ecs - name: threat.indicator.url.query - ignore_above: 4096 -- external: ecs - name: threat.indicator.url.registered_domain -- external: ecs - name: threat.indicator.url.scheme -- external: ecs - name: threat.indicator.url.subdomain -- external: ecs - name: threat.indicator.url.top_level_domain -- external: ecs - name: threat.indicator.url.username -- external: ecs - name: threat.group.id -- external: ecs - name: threat.group.name -- external: ecs - name: threat.software.type -- external: ecs - name: threat.software.name -- external: ecs - name: threat.feed.name -- external: ecs - name: threat.indicator.marking.tlp -- external: ecs - name: threat.indicator.marking.tlp_version -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type diff --git a/packages/ti_mandiant_advantage/data_stream/threat_intelligence/sample_event.json b/packages/ti_mandiant_advantage/data_stream/threat_intelligence/sample_event.json index 92a0a1a0a7d..6f5268c698e 100644 --- a/packages/ti_mandiant_advantage/data_stream/threat_intelligence/sample_event.json +++ b/packages/ti_mandiant_advantage/data_stream/threat_intelligence/sample_event.json @@ -1,7 +1,7 @@ { "@timestamp": "2023-05-05T15:45:59.710Z", "ecs": { - "version": "8.7.0" + "version": "8.11.0" }, "event": { "category": [ @@ -9,7 +9,7 @@ ], "kind": "enrichment", "module": "ti_mandiant_advantage_threat_intelligence", - "risk_score": 50.0, + "risk_score": 50, "type": [ "indicator" ] diff --git a/packages/ti_mandiant_advantage/docs/README.md b/packages/ti_mandiant_advantage/docs/README.md index 3e1267d9baf..9cc42076a82 100644 --- a/packages/ti_mandiant_advantage/docs/README.md +++ b/packages/ti_mandiant_advantage/docs/README.md @@ -59,7 +59,7 @@ An example event for `threat_intelligence` looks as following: { "@timestamp": "2023-05-05T15:45:59.710Z", "ecs": { - "version": "8.7.0" + "version": "8.11.0" }, "event": { "category": [ @@ -67,7 +67,7 @@ An example event for `threat_intelligence` looks as following: ], "kind": "enrichment", "module": "ti_mandiant_advantage_threat_intelligence", - "risk_score": 50.0, + "risk_score": 50, "type": [ "indicator" ] @@ -168,46 +168,13 @@ An example event for `threat_intelligence` looks as following: |---|---|---| | @timestamp | Event timestamp. | date | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.risk_score | Risk score or priority of the event (e.g. security solutions). Use your system's original value here. | float | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | mandiant.threat_intelligence.ioc.associated_hashes | List of associated hashes and their types. | object | @@ -222,54 +189,4 @@ An example event for `threat_intelligence` looks as following: | mandiant.threat_intelligence.ioc.sources | List of the indicator sources. | object | | mandiant.threat_intelligence.ioc.type | IOC type. | keyword | | mandiant.threat_intelligence.ioc.value | IOC value. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| tags | List of keywords used to tag each event. | keyword | -| threat.feed.name | The name of the threat feed in UI friendly format. | keyword | -| threat.group.id | The id of the group for a set of related intrusion activity that are tracked by a common name in the security community. While not required, you can use a MITRE ATT&CK® group id. | keyword | -| threat.group.name | The name of the group for a set of related intrusion activity that are tracked by a common name in the security community. While not required, you can use a MITRE ATT&CK® group name. | keyword | -| threat.indicator.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| threat.indicator.as.organization.name | Organization name. | keyword | -| threat.indicator.as.organization.name.text | Multi-field of `threat.indicator.as.organization.name`. | match_only_text | -| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword | -| threat.indicator.description | Describes the type of action conducted by the threat. | keyword | -| threat.indicator.email.address | Identifies a threat indicator as an email address (irrespective of direction). | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.file.hash.sha384 | SHA384 hash. | keyword | -| threat.indicator.file.hash.sha512 | SHA512 hash. | keyword | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.geo.city_name | City name. | keyword | -| threat.indicator.geo.continent_name | Name of the continent. | keyword | -| threat.indicator.geo.country_iso_code | Country ISO code. | keyword | -| threat.indicator.geo.country_name | Country name. | keyword | -| threat.indicator.geo.location | Longitude and latitude. | geo_point | -| threat.indicator.geo.region_iso_code | Region ISO code. | keyword | -| threat.indicator.geo.region_name | Region name. | keyword | -| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.marking.tlp | Traffic Light Protocol sharing markings. | keyword | -| threat.indicator.marking.tlp_version | Traffic Light Protocol version. | keyword | -| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| threat.indicator.url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | -| threat.indicator.url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| threat.indicator.url.full.text | Multi-field of `threat.indicator.url.full`. | match_only_text | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | -| threat.indicator.url.password | Password of the request. | keyword | -| threat.indicator.url.path | Path of the request, such as "/search". | wildcard | -| threat.indicator.url.port | Port of the request, such as 443. | long | -| threat.indicator.url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| threat.indicator.url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| threat.indicator.url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| threat.indicator.url.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| threat.indicator.url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| threat.indicator.url.username | Username of the request. | keyword | -| threat.software.name | The name of the software used by this threat to conduct behavior commonly modeled using MITRE ATT&CK®. While not required, you can use a MITRE ATT&CK® software name. | keyword | -| threat.software.type | The type of software used by this threat to conduct behavior commonly modeled using MITRE ATT&CK®. While not required, you can use a MITRE ATT&CK® software type. | keyword | diff --git a/packages/ti_mandiant_advantage/manifest.yml b/packages/ti_mandiant_advantage/manifest.yml index 1f6d30be0d8..c3414188796 100644 --- a/packages/ti_mandiant_advantage/manifest.yml +++ b/packages/ti_mandiant_advantage/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.2 name: ti_mandiant_advantage title: "Mandiant Advantage" -version: 1.2.0 +version: "1.3.0" source: license: "Elastic-2.0" description: "Collect Threat Intelligence from products within the Mandiant Advantage platform." @@ -11,7 +11,7 @@ categories: - security conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" elastic: subscription: basic screenshots: From 531d0ddb1d91e4b513683609f268d87ec6f8c729 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:28:01 +0930 Subject: [PATCH 102/121] [ti_misp] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/ti_misp --- packages/ti_misp/changelog.yml | 5 + .../data_stream/threat/fields/agent.yml | 167 +----------------- .../data_stream/threat/fields/beats.yml | 3 - .../ti_misp/data_stream/threat/fields/ecs.yml | 76 -------- .../threat_attributes/fields/agent.yml | 167 +----------------- .../threat_attributes/fields/beats.yml | 3 - .../threat_attributes/fields/ecs.yml | 80 --------- packages/ti_misp/docs/README.md | 136 -------------- packages/ti_misp/manifest.yml | 4 +- 9 files changed, 9 insertions(+), 632 deletions(-) delete mode 100644 packages/ti_misp/data_stream/threat/fields/ecs.yml diff --git a/packages/ti_misp/changelog.yml b/packages/ti_misp/changelog.yml index 85377169535..9d2f13560cd 100644 --- a/packages/ti_misp/changelog.yml +++ b/packages/ti_misp/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.35.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.34.0" changes: - description: Allow user configuration of event limit in threat data stream. diff --git a/packages/ti_misp/data_stream/threat/fields/agent.yml b/packages/ti_misp/data_stream/threat/fields/agent.yml index da4e652c53b..2bc58530bac 100644 --- a/packages/ti_misp/data_stream/threat/fields/agent.yml +++ b/packages/ti_misp/data_stream/threat/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/ti_misp/data_stream/threat/fields/beats.yml b/packages/ti_misp/data_stream/threat/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/ti_misp/data_stream/threat/fields/beats.yml +++ b/packages/ti_misp/data_stream/threat/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/ti_misp/data_stream/threat/fields/ecs.yml b/packages/ti_misp/data_stream/threat/fields/ecs.yml deleted file mode 100644 index 31cdaf0274f..00000000000 --- a/packages/ti_misp/data_stream/threat/fields/ecs.yml +++ /dev/null @@ -1,76 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: tags -- external: ecs - name: error.message -- external: ecs - name: event.category -- external: ecs - name: event.ingested -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.type -- external: ecs - name: event.original -- external: ecs - name: user.email -- external: ecs - name: user.roles -- external: ecs - name: threat.indicator.first_seen -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.indicator.scanner_stats -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.ip -- external: ecs - name: threat.indicator.url.domain -- external: ecs - name: threat.indicator.url.full -- external: ecs - name: threat.indicator.url.extension -- external: ecs - name: threat.indicator.url.original -- external: ecs - name: threat.indicator.url.path -- external: ecs - name: threat.indicator.url.port -- external: ecs - name: threat.indicator.url.scheme -- external: ecs - name: threat.indicator.url.query -- external: ecs - name: threat.indicator.email.address -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.as.number -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.marking.tlp -- external: ecs - name: threat.indicator.port -- external: ecs - name: threat.indicator.registry.key -- external: ecs - name: threat.indicator.registry.value -- external: ecs - name: threat.indicator.file.size -- external: ecs - name: threat.indicator.file.type -- external: ecs - name: threat.indicator.file.name diff --git a/packages/ti_misp/data_stream/threat_attributes/fields/agent.yml b/packages/ti_misp/data_stream/threat_attributes/fields/agent.yml index da4e652c53b..2bc58530bac 100644 --- a/packages/ti_misp/data_stream/threat_attributes/fields/agent.yml +++ b/packages/ti_misp/data_stream/threat_attributes/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/ti_misp/data_stream/threat_attributes/fields/beats.yml b/packages/ti_misp/data_stream/threat_attributes/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/ti_misp/data_stream/threat_attributes/fields/beats.yml +++ b/packages/ti_misp/data_stream/threat_attributes/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/ti_misp/data_stream/threat_attributes/fields/ecs.yml b/packages/ti_misp/data_stream/threat_attributes/fields/ecs.yml index a9f7e59c644..258b3f57396 100644 --- a/packages/ti_misp/data_stream/threat_attributes/fields/ecs.yml +++ b/packages/ti_misp/data_stream/threat_attributes/fields/ecs.yml @@ -1,82 +1,2 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: tags -- external: ecs - name: error.message -- external: ecs - name: event.category -- external: ecs - name: event.ingested -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.type -- external: ecs - name: event.original -- external: ecs - name: user.email -- external: ecs - name: user.roles -- external: ecs - name: threat.indicator.first_seen -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.indicator.scanner_stats -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.ip -- external: ecs - name: threat.indicator.url.domain -- external: ecs - name: threat.indicator.url.full -- external: ecs - name: threat.indicator.url.extension -- external: ecs - name: threat.indicator.url.original -- external: ecs - name: threat.indicator.url.path -- external: ecs - name: threat.indicator.url.port -- external: ecs - name: threat.indicator.url.scheme -- external: ecs - name: threat.indicator.url.query -- external: ecs - name: threat.indicator.email.address -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.as.number -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.marking.tlp -- external: ecs - name: threat.indicator.port -- external: ecs - name: threat.indicator.registry.key -- external: ecs - name: threat.indicator.registry.value -- external: ecs - name: threat.indicator.file.size -- external: ecs - name: threat.indicator.file.type -- external: ecs - name: threat.indicator.file.name -- external: ecs - name: organization.id -- external: ecs - name: labels - name: threat.indicator.email.subject type: keyword diff --git a/packages/ti_misp/docs/README.md b/packages/ti_misp/docs/README.md index 497af09798c..182de98481b 100644 --- a/packages/ti_misp/docs/README.md +++ b/packages/ti_misp/docs/README.md @@ -16,54 +16,18 @@ The filters themselves are based on the [MISP API documentation](https://www.cir | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | misp.attribute.category | The category of the attribute related to the event object. For example "Network Activity". | keyword | | misp.attribute.comment | Comments made to the attribute itself. | keyword | | misp.attribute.deleted | If the attribute has been removed from the event object. | boolean | @@ -137,39 +101,8 @@ The filters themselves are based on the [MISP API documentation](https://www.cir | misp.orgc.local | If the Organization Community was local or synced from a remote source. | boolean | | misp.orgc.name | The Organization Community name in which the event object was reported from. | keyword | | misp.orgc.uuid | The Organization Community UUID in which the event object was reported from. | keyword | -| tags | List of keywords used to tag each event. | keyword | | threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword | | threat.feed.name | Display friendly feed name. | constant_keyword | -| threat.indicator.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| threat.indicator.email.address | Identifies a threat indicator as an email address (irrespective of direction). | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.file.name | Name of the file including the extension, without the directory. | keyword | -| threat.indicator.file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| threat.indicator.file.type | File type (file, dir, or symlink). | keyword | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.marking.tlp | Traffic Light Protocol sharing markings. | keyword | -| threat.indicator.port | Identifies a threat indicator as a port number (irrespective of direction). | long | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.registry.key | Hive-relative path of keys. | keyword | -| threat.indicator.registry.value | Name of the value written. | keyword | -| threat.indicator.scanner_stats | Count of AV/EDR vendors that successfully detected malicious file or URL. | long | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| threat.indicator.url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| threat.indicator.url.full.text | Multi-field of `threat.indicator.url.full`. | match_only_text | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | -| threat.indicator.url.path | Path of the request, such as "/search". | wildcard | -| threat.indicator.url.port | Port of the request, such as 443. | long | -| threat.indicator.url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| threat.indicator.url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| user.email | User email address. | keyword | -| user.roles | Array of user roles at the time of the event. | keyword | An example event for `threat` looks as following: @@ -303,56 +236,19 @@ To facilitate IOC expiration, source datastream-backed indices `.ds-logs-ti_misp | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| labels | Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels. | object | | labels.is_ioc_transform_source | Field indicating if the document is a source for the transform. This field is not added to destination indices to facilitate easier filtering of indicators for indicator match rules. | constant_keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | misp.attribute.category | The category of the attribute. For example "Network Activity". | keyword | | misp.attribute.comment | Comments made to the attribute itself. | keyword | | misp.attribute.data | The data of the attribute | keyword | @@ -406,40 +302,8 @@ To facilitate IOC expiration, source datastream-backed indices `.ds-logs-ti_misp | misp.object.template_version | The version of attribute object's template. | keyword | | misp.object.timestamp | The timestamp when the object was created. | date | | misp.object.uuid | The UUID of the object in which the attribute is attached. | keyword | -| organization.id | Unique identifier for the organization. | keyword | -| tags | List of keywords used to tag each event. | keyword | | threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword | | threat.feed.name | Display friendly feed name | constant_keyword | -| threat.indicator.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| threat.indicator.email.address | Identifies a threat indicator as an email address (irrespective of direction). | keyword | | threat.indicator.email.subject | | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.file.name | Name of the file including the extension, without the directory. | keyword | -| threat.indicator.file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| threat.indicator.file.type | File type (file, dir, or symlink). | keyword | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.marking.tlp | Traffic Light Protocol sharing markings. | keyword | -| threat.indicator.port | Identifies a threat indicator as a port number (irrespective of direction). | long | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.registry.key | Hive-relative path of keys. | keyword | -| threat.indicator.registry.value | Name of the value written. | keyword | -| threat.indicator.scanner_stats | Count of AV/EDR vendors that successfully detected malicious file or URL. | long | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| threat.indicator.url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| threat.indicator.url.full.text | Multi-field of `threat.indicator.url.full`. | match_only_text | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | -| threat.indicator.url.path | Path of the request, such as "/search". | wildcard | -| threat.indicator.url.port | Port of the request, such as 443. | long | -| threat.indicator.url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| threat.indicator.url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| user.email | User email address. | keyword | -| user.roles | Array of user roles at the time of the event. | keyword | diff --git a/packages/ti_misp/manifest.yml b/packages/ti_misp/manifest.yml index 5abf8aec31c..112529722fd 100644 --- a/packages/ti_misp/manifest.yml +++ b/packages/ti_misp/manifest.yml @@ -1,13 +1,13 @@ name: ti_misp title: MISP -version: "1.34.0" +version: "1.35.0" description: Ingest threat intelligence indicators from MISP platform with Elastic Agent. type: integration format_version: "3.0.2" categories: ["security", "threat_intel"] conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" icons: - src: /img/misp.svg title: MISP From 6ebf32f715dbce6c3c3dc524634e97a24abb8523 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:28:03 +0930 Subject: [PATCH 103/121] [tines] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/tines --- packages/tines/changelog.yml | 5 + .../data_stream/audit_logs/fields/ecs.yml | 110 ---------------- .../data_stream/time_saved/fields/ecs.yml | 110 ---------------- packages/tines/docs/README.md | 124 ------------------ packages/tines/manifest.yml | 4 +- 5 files changed, 7 insertions(+), 346 deletions(-) diff --git a/packages/tines/changelog.yml b/packages/tines/changelog.yml index 65628977b24..aad7971250e 100644 --- a/packages/tines/changelog.yml +++ b/packages/tines/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.12.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.11.0" changes: - description: Update manifest format version to v3.0.3. diff --git a/packages/tines/data_stream/audit_logs/fields/ecs.yml b/packages/tines/data_stream/audit_logs/fields/ecs.yml index 170800e18db..b7e148e95ed 100644 --- a/packages/tines/data_stream/audit_logs/fields/ecs.yml +++ b/packages/tines/data_stream/audit_logs/fields/ecs.yml @@ -6,113 +6,3 @@ name: data_stream.dataset - external: ecs name: data_stream.namespace -- external: ecs - name: message -- external: ecs - name: 'tags' -- external: ecs - name: ecs.version -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.code -- external: ecs - name: event.created -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.outcome -- external: ecs - name: event.original -- external: ecs - name: event.provider -- external: ecs - name: event.sequence -- external: ecs - name: event.type -- external: ecs - name: event.id -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: source.user.domain -- external: ecs - name: source.user.id -- external: ecs - name: source.user.name -- external: ecs - name: user.domain -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user.email -- external: ecs - name: source.ip -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.name -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.version -- external: ecs - name: user_agent.os.family -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.kernel -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.platform -- external: ecs - name: user_agent.os.type -- external: ecs - name: user_agent.os.version -- external: ecs - name: url.original -- external: ecs - name: url.domain -- external: ecs - name: url.extension -- external: ecs - name: url.path -- external: ecs - name: url.port -- external: ecs - name: url.registered_domain -- external: ecs - name: url.scheme -- external: ecs - name: url.top_level_domain diff --git a/packages/tines/data_stream/time_saved/fields/ecs.yml b/packages/tines/data_stream/time_saved/fields/ecs.yml index 170800e18db..b7e148e95ed 100644 --- a/packages/tines/data_stream/time_saved/fields/ecs.yml +++ b/packages/tines/data_stream/time_saved/fields/ecs.yml @@ -6,113 +6,3 @@ name: data_stream.dataset - external: ecs name: data_stream.namespace -- external: ecs - name: message -- external: ecs - name: 'tags' -- external: ecs - name: ecs.version -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.code -- external: ecs - name: event.created -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.outcome -- external: ecs - name: event.original -- external: ecs - name: event.provider -- external: ecs - name: event.sequence -- external: ecs - name: event.type -- external: ecs - name: event.id -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: source.user.domain -- external: ecs - name: source.user.id -- external: ecs - name: source.user.name -- external: ecs - name: user.domain -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user.email -- external: ecs - name: source.ip -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.geo.name -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.version -- external: ecs - name: user_agent.os.family -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.kernel -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.platform -- external: ecs - name: user_agent.os.type -- external: ecs - name: user_agent.os.version -- external: ecs - name: url.original -- external: ecs - name: url.domain -- external: ecs - name: url.extension -- external: ecs - name: url.path -- external: ecs - name: url.port -- external: ecs - name: url.registered_domain -- external: ecs - name: url.scheme -- external: ecs - name: url.top_level_domain diff --git a/packages/tines/docs/README.md b/packages/tines/docs/README.md index 7a0c08aca50..7242b575738 100644 --- a/packages/tines/docs/README.md +++ b/packages/tines/docs/README.md @@ -70,41 +70,7 @@ All fields ingested to this data stream are stored under `tines.audit_log` as ea | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.sequence | Sequence number of the event. The sequence number is a value published by some event sources, to make the exact ordering of events unambiguous, regardless of the timestamp precision. | long | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | input.type | | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| source.user.id | Unique identifier of the user. | keyword | -| source.user.name | Short name or login of the user. | keyword | -| source.user.name.text | Multi-field of `source.user.name`. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | | tines.audit_log.created_at | The date and time that the audit log event occurred | date | | tines.audit_log.id | A unique ID for the audit log event | long | | tines.audit_log.inputs.actionIds | | long | @@ -172,34 +138,6 @@ All fields ingested to this data stream are stored under `tines.audit_log` as ea | tines.audit_log.user_id | The ID of the user who triggered the operation | long | | tines.audit_log.user_name | The name of the user who triggered the operation | keyword | | tines.tenant_url | The tenant URL associated that provided the event | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | -| url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.kernel | Operating system kernel version as a raw string. | keyword | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| user_agent.os.type | Use the `os.type` field to categorize the operating system into one of the broad commercial families. If the OS you're dealing with is not listed as an expected value, the field should not be populated. Please let us know by opening an issue with ECS, to propose its addition. | keyword | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | An example event for `audit` looks as following: @@ -345,74 +283,12 @@ All fields ingested to this data stream are stored under `tines.time_saved` as e | data_stream.dataset | The field can contain anything that makes sense to signify the source of the data. Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`. Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | | data_stream.type | An overarching type for the data stream. Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.sequence | Sequence number of the event. The sequence number is a value published by some event sources, to make the exact ordering of events unambiguous, regardless of the timestamp precision. | long | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | input.type | | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| source.user.id | Unique identifier of the user. | keyword | -| source.user.name | Short name or login of the user. | keyword | -| source.user.name.text | Multi-field of `source.user.name`. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | | tines.tenant_url | The tenant URL associated that provided the event | keyword | | tines.time_saved.date | The date and time for the time saved period | date | | tines.time_saved.story_id | Story ID for time saved | long | | tines.time_saved.team_id | Team ID for time saved | long | | tines.time_saved.value | Time saved in seconds | long | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | -| url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.email | User email address. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.kernel | Operating system kernel version as a raw string. | keyword | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| user_agent.os.type | Use the `os.type` field to categorize the operating system into one of the broad commercial families. If the OS you're dealing with is not listed as an expected value, the field should not be populated. Please let us know by opening an issue with ECS, to propose its addition. | keyword | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | An example event for `time_saved` looks as following: diff --git a/packages/tines/manifest.yml b/packages/tines/manifest.yml index f16eef73735..e78276a97d3 100644 --- a/packages/tines/manifest.yml +++ b/packages/tines/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: tines title: "Tines" -version: "1.11.0" +version: "1.12.0" description: "Tines Logs & Time Saved Reports" type: integration categories: @@ -9,7 +9,7 @@ categories: - security conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" elastic: subscription: "basic" screenshots: From 8b8b79aafb9d3721bb4509038afe7be4fa2fc792 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:28:06 +0930 Subject: [PATCH 104/121] [ti_opencti] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/ti_opencti --- packages/ti_opencti/_dev/build/build.yml | 1 - packages/ti_opencti/changelog.yml | 5 + .../data_stream/indicator/fields/ecs.yml | 166 ------------------ packages/ti_opencti/docs/README.md | 84 --------- packages/ti_opencti/manifest.yml | 4 +- 5 files changed, 7 insertions(+), 253 deletions(-) diff --git a/packages/ti_opencti/_dev/build/build.yml b/packages/ti_opencti/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/ti_opencti/_dev/build/build.yml +++ b/packages/ti_opencti/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/ti_opencti/changelog.yml b/packages/ti_opencti/changelog.yml index ba759a88b85..124b1855d9e 100644 --- a/packages/ti_opencti/changelog.yml +++ b/packages/ti_opencti/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.3.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.2.0" changes: - description: Extend `threat.indicator.type` definition to allow ECS conformance. diff --git a/packages/ti_opencti/data_stream/indicator/fields/ecs.yml b/packages/ti_opencti/data_stream/indicator/fields/ecs.yml index b7f2d2f1a58..f6d9a652390 100644 --- a/packages/ti_opencti/data_stream/indicator/fields/ecs.yml +++ b/packages/ti_opencti/data_stream/indicator/fields/ecs.yml @@ -1,8 +1,4 @@ # Manually define these as a workaround for failing expected values validation -- name: threat.indicator.name - level: extended - type: keyword - description: The display name indicator in an UI friendly format - name: threat.indicator.type level: extended type: keyword @@ -39,174 +35,12 @@ - x509-certificate - unknown - port -# Additional file hash algorithms - name: threat.indicator.file.hash.sha3_256 type: keyword description: SHA3-256 hash. - name: threat.indicator.file.hash.sha3_512 type: keyword description: SHA3-512 hash. -# External ECS defintions, required by the transform -- external: ecs - name: ecs.version -- external: ecs - name: event.agent_id_status -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.dataset -- external: ecs - name: event.id -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: tags -- external: ecs - name: threat.feed.dashboard_id -- external: ecs - name: threat.feed.description -- external: ecs - name: threat.feed.name -- external: ecs - name: threat.feed.reference -- external: ecs - name: threat.indicator.as.number -- external: ecs - name: threat.indicator.as.organization.name -- external: ecs - name: threat.indicator.confidence -- external: ecs - name: threat.indicator.description -- external: ecs - name: threat.indicator.email.address -- external: ecs - name: threat.indicator.file.accessed -- external: ecs - name: threat.indicator.file.created -- external: ecs - name: threat.indicator.file.directory -- external: ecs - name: threat.indicator.file.drive_letter -- external: ecs - name: threat.indicator.file.extension -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.file.hash.sha384 -- external: ecs - name: threat.indicator.file.hash.sha512 -- external: ecs - name: threat.indicator.file.hash.ssdeep -- external: ecs - name: threat.indicator.file.hash.tlsh -- external: ecs - name: threat.indicator.file.mime_type -- external: ecs - name: threat.indicator.file.mtime -- external: ecs - name: threat.indicator.file.name -- external: ecs - name: threat.indicator.file.path -- external: ecs - name: threat.indicator.file.size -- external: ecs - name: threat.indicator.file.type -- external: ecs - name: threat.indicator.ip -- external: ecs - name: threat.indicator.marking.tlp -- external: ecs - name: threat.indicator.modified_at -- external: ecs - name: threat.indicator.port -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.reference -- external: ecs - name: threat.indicator.registry.data.bytes -- external: ecs - name: threat.indicator.registry.data.strings -- external: ecs - name: threat.indicator.registry.data.type -- external: ecs - name: threat.indicator.registry.hive -- external: ecs - name: threat.indicator.registry.key -- external: ecs - name: threat.indicator.registry.path -- external: ecs - name: threat.indicator.registry.value -- external: ecs - name: threat.indicator.url.domain -- external: ecs - name: threat.indicator.url.extension -- external: ecs - name: threat.indicator.url.fragment -- external: ecs - name: threat.indicator.url.full -- external: ecs - name: threat.indicator.url.original -- external: ecs - name: threat.indicator.url.password -- external: ecs - name: threat.indicator.url.path -- external: ecs - name: threat.indicator.url.port -- external: ecs - name: threat.indicator.url.query -- external: ecs - name: threat.indicator.url.registered_domain -- external: ecs - name: threat.indicator.url.scheme -- external: ecs - name: threat.indicator.url.subdomain -- external: ecs - name: threat.indicator.url.top_level_domain -- external: ecs - name: threat.indicator.url.username -- external: ecs - name: threat.indicator.x509.alternative_names -- external: ecs - name: threat.indicator.x509.issuer.common_name -- external: ecs - name: threat.indicator.x509.not_after -- external: ecs - name: threat.indicator.x509.not_before -- external: ecs - name: threat.indicator.x509.public_key_algorithm -- external: ecs - name: threat.indicator.x509.public_key_exponent -- external: ecs - name: threat.indicator.x509.serial_number -- external: ecs - name: threat.indicator.x509.signature_algorithm -- external: ecs - name: threat.indicator.x509.subject.common_name -- external: ecs - name: threat.indicator.x509.version_number -# Below fields to be moved into base-fields.yml after kibana.version changed to >= 8.14 -# Related to fix: https://github.com/elastic/kibana/pull/177608 - name: event.module type: constant_keyword description: Event module diff --git a/packages/ti_opencti/docs/README.md b/packages/ti_opencti/docs/README.md index ede58b254d7..c8884ecc123 100644 --- a/packages/ti_opencti/docs/README.md +++ b/packages/ti_opencti/docs/README.md @@ -193,17 +193,7 @@ The documentation for ECS fields can be found at: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.agent_id_status | Agents are normally responsible for populating the `agent.id` field value. If the system receiving events is capable of validating the value based on authentication information for the client then this field can be used to reflect the outcome of that validation. For example if the agent's connection is authenticated with mTLS and the client cert contains the ID of the agent to which the cert was issued then the `agent.id` value in events can be checked against the certificate. If the values match then `event.agent_id_status: verified` is added to the event, otherwise one of the other allowed values should be used. If no validation is performed then the field should be omitted. The allowed values are: `verified` - The `agent.id` field value matches expected value obtained from auth metadata. `mismatch` - The `agent.id` field value does not match the expected value obtained from auth metadata. `missing` - There was no `agent.id` field in the event to validate. `auth_metadata_missing` - There was no auth metadata or it was missing information about the agent ID. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | input.type | Input type. | keyword | | labels.is_ioc_transform_source | Field indicating if the document is a source for the transform. This field is not added to destination indices to facilitate easier filtering of indicators for indicator match rules. | constant_keyword | | opencti.indicator.creator_identity_class | The type of the creator of this indicator (e.g. "organization"). | keyword | @@ -521,81 +511,7 @@ The documentation for ECS fields can be found at: | opencti.observable.x509_certificate.validity_not_before | The date on which the certificate validity period begins. | date | | opencti.observable.x509_certificate.value | The main value for the observable. | keyword | | opencti.observable.x509_certificate.version | The version of the encoded certificate. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| threat.feed.dashboard_id | The saved object ID of the dashboard belonging to the threat feed for displaying dashboard links to threat feeds in Kibana. | keyword | -| threat.feed.description | Description of the threat feed in a UI friendly format. | keyword | -| threat.feed.name | The name of the threat feed in UI friendly format. | keyword | -| threat.feed.reference | Reference information for the threat feed in a UI friendly format. | keyword | -| threat.indicator.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| threat.indicator.as.organization.name | Organization name. | keyword | -| threat.indicator.as.organization.name.text | Multi-field of `threat.indicator.as.organization.name`. | match_only_text | -| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword | -| threat.indicator.description | Describes the type of action conducted by the threat. | keyword | -| threat.indicator.email.address | Identifies a threat indicator as an email address (irrespective of direction). | keyword | -| threat.indicator.file.accessed | Last time the file was accessed. Note that not all filesystems keep track of access time. | date | -| threat.indicator.file.created | File creation time. Note that not all filesystems store the creation time. | date | -| threat.indicator.file.directory | Directory where the file is located. It should include the drive letter, when appropriate. | keyword | -| threat.indicator.file.drive_letter | Drive letter where the file is located. This field is only relevant on Windows. The value should be uppercase, and not include the colon. | keyword | -| threat.indicator.file.extension | File extension, excluding the leading dot. Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.file.hash.sha384 | SHA384 hash. | keyword | | threat.indicator.file.hash.sha3_256 | SHA3-256 hash. | keyword | | threat.indicator.file.hash.sha3_512 | SHA3-512 hash. | keyword | -| threat.indicator.file.hash.sha512 | SHA512 hash. | keyword | -| threat.indicator.file.hash.ssdeep | SSDEEP hash. | keyword | -| threat.indicator.file.hash.tlsh | TLSH hash. | keyword | -| threat.indicator.file.mime_type | MIME type should identify the format of the file or stream of bytes using https://www.iana.org/assignments/media-types/media-types.xhtml[IANA official types], where possible. When more than one type is applicable, the most specific type should be used. | keyword | -| threat.indicator.file.mtime | Last time the file content was modified. | date | -| threat.indicator.file.name | Name of the file including the extension, without the directory. | keyword | -| threat.indicator.file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| threat.indicator.file.path.text | Multi-field of `threat.indicator.file.path`. | match_only_text | -| threat.indicator.file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| threat.indicator.file.type | File type (file, dir, or symlink). | keyword | -| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | -| threat.indicator.marking.tlp | Traffic Light Protocol sharing markings. | keyword | -| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | -| threat.indicator.name | The display name indicator in an UI friendly format | keyword | -| threat.indicator.port | Identifies a threat indicator as a port number (irrespective of direction). | long | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.reference | Reference URL linking to additional information about this indicator. | keyword | -| threat.indicator.registry.data.bytes | Original bytes written with base64 encoding. For Windows registry operations, such as SetValueEx and RegQueryValueEx, this corresponds to the data pointed by `lp_data`. This is optional but provides better recoverability and should be populated for REG_BINARY encoded values. | keyword | -| threat.indicator.registry.data.strings | Content when writing string types. Populated as an array when writing string data to the registry. For single string registry types (REG_SZ, REG_EXPAND_SZ), this should be an array with one string. For sequences of string with REG_MULTI_SZ, this array will be variable length. For numeric data, such as REG_DWORD and REG_QWORD, this should be populated with the decimal representation (e.g `"1"`). | wildcard | -| threat.indicator.registry.data.type | Standard registry type for encoding contents | keyword | -| threat.indicator.registry.hive | Abbreviated name for the hive. | keyword | -| threat.indicator.registry.key | Hive-relative path of keys. | keyword | -| threat.indicator.registry.path | Full path, including hive, key and value | keyword | -| threat.indicator.registry.value | Name of the value written. | keyword | | threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.1 or OpenCTI | keyword | -| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| threat.indicator.url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | -| threat.indicator.url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| threat.indicator.url.full.text | Multi-field of `threat.indicator.url.full`. | match_only_text | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | -| threat.indicator.url.password | Password of the request. | keyword | -| threat.indicator.url.path | Path of the request, such as "/search". | wildcard | -| threat.indicator.url.port | Port of the request, such as 443. | long | -| threat.indicator.url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| threat.indicator.url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| threat.indicator.url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| threat.indicator.url.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| threat.indicator.url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| threat.indicator.url.username | Username of the request. | keyword | -| threat.indicator.x509.alternative_names | List of subject alternative names (SAN). Name types vary by certificate authority and certificate type but commonly contain IP addresses, DNS names (and wildcards), and email addresses. | keyword | -| threat.indicator.x509.issuer.common_name | List of common name (CN) of issuing certificate authority. | keyword | -| threat.indicator.x509.not_after | Time at which the certificate is no longer considered valid. | date | -| threat.indicator.x509.not_before | Time at which the certificate is first considered valid. | date | -| threat.indicator.x509.public_key_algorithm | Algorithm used to generate the public key. | keyword | -| threat.indicator.x509.public_key_exponent | Exponent used to derive the public key. This is algorithm specific. | long | -| threat.indicator.x509.serial_number | Unique serial number issued by the certificate authority. For consistency, if this value is alphanumeric, it should be formatted without colons and uppercase characters. | keyword | -| threat.indicator.x509.signature_algorithm | Identifier for certificate signature algorithm. We recommend using names found in Go Lang Crypto library. See https://github.com/golang/go/blob/go1.14/src/crypto/x509/x509.go#L337-L353. | keyword | -| threat.indicator.x509.subject.common_name | List of common names (CN) of subject. | keyword | -| threat.indicator.x509.version_number | Version of x509 format. | keyword | diff --git a/packages/ti_opencti/manifest.yml b/packages/ti_opencti/manifest.yml index cd743ee76bf..58d590fa9fd 100644 --- a/packages/ti_opencti/manifest.yml +++ b/packages/ti_opencti/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: ti_opencti title: OpenCTI -version: "2.2.0" +version: "2.3.0" description: "Ingest threat intelligence indicators from OpenCTI with Elastic Agent." type: integration source: @@ -11,7 +11,7 @@ categories: - threat_intel conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" screenshots: - src: /img/screenshot1.png title: "Dashboard: OpenCTI Overview" From 5e7cb605bf4e37581c77ac76d3248cab2bff2fc8 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:28:08 +0930 Subject: [PATCH 105/121] [ti_otx] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/ti_otx --- packages/ti_otx/changelog.yml | 5 + .../pulses_subscribed/fields/agent.yml | 50 ------ .../pulses_subscribed/fields/beats.yml | 3 - .../pulses_subscribed/fields/ecs.yml | 58 ------ .../data_stream/threat/fields/agent.yml | 167 +----------------- .../data_stream/threat/fields/beats.yml | 3 - .../ti_otx/data_stream/threat/fields/ecs.yml | 58 ------ packages/ti_otx/docs/README.md | 116 ------------ packages/ti_otx/manifest.yml | 4 +- 9 files changed, 8 insertions(+), 456 deletions(-) delete mode 100644 packages/ti_otx/data_stream/pulses_subscribed/fields/agent.yml diff --git a/packages/ti_otx/changelog.yml b/packages/ti_otx/changelog.yml index f9f9c620b70..1355341f819 100644 --- a/packages/ti_otx/changelog.yml +++ b/packages/ti_otx/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.25.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.24.1" changes: - description: Fix type-mapping inconsistency for `otx.id` field. diff --git a/packages/ti_otx/data_stream/pulses_subscribed/fields/agent.yml b/packages/ti_otx/data_stream/pulses_subscribed/fields/agent.yml deleted file mode 100644 index 8cf2742f52d..00000000000 --- a/packages/ti_otx/data_stream/pulses_subscribed/fields/agent.yml +++ /dev/null @@ -1,50 +0,0 @@ -- name: cloud.account.id - external: ecs -- name: cloud.availability_zone - external: ecs -- name: cloud.instance.id - external: ecs -- name: cloud.instance.name - external: ecs -- name: cloud.machine.type - external: ecs -- name: cloud.provider - external: ecs -- name: cloud.region - external: ecs -- name: cloud.project.id - external: ecs -- name: container.id - external: ecs -- name: container.image.name - external: ecs -- name: container.labels - external: ecs -- name: container.name - external: ecs -- name: host.architecture - external: ecs -- name: host.domain - external: ecs -- name: host.hostname - external: ecs -- name: host.id - external: ecs -- name: host.ip - external: ecs -- name: host.mac - external: ecs -- name: host.name - external: ecs -- name: host.os.family - external: ecs -- name: host.os.kernel - external: ecs -- name: host.os.name - external: ecs -- name: host.os.platform - external: ecs -- name: host.os.version - external: ecs -- name: host.type - external: ecs diff --git a/packages/ti_otx/data_stream/pulses_subscribed/fields/beats.yml b/packages/ti_otx/data_stream/pulses_subscribed/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/ti_otx/data_stream/pulses_subscribed/fields/beats.yml +++ b/packages/ti_otx/data_stream/pulses_subscribed/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/ti_otx/data_stream/pulses_subscribed/fields/ecs.yml b/packages/ti_otx/data_stream/pulses_subscribed/fields/ecs.yml index 97db980d239..34fc117cd80 100644 --- a/packages/ti_otx/data_stream/pulses_subscribed/fields/ecs.yml +++ b/packages/ti_otx/data_stream/pulses_subscribed/fields/ecs.yml @@ -1,61 +1,3 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: error.message -- external: ecs - name: tags -- external: ecs - name: related.hash -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.created -- external: ecs - name: event.category -- external: ecs - name: event.type -- external: ecs - name: event.original -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.email.address -- external: ecs - name: threat.indicator.ip -- external: ecs - name: threat.indicator.url.domain -- external: ecs - name: threat.indicator.url.full -- external: ecs - name: threat.indicator.url.extension -- external: ecs - name: threat.indicator.url.original -- external: ecs - name: threat.indicator.url.path -- external: ecs - name: threat.indicator.url.port -- external: ecs - name: threat.indicator.url.scheme -- external: ecs - name: threat.indicator.url.query -- external: ecs - name: threat.indicator.file.type -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.file.pe.imphash - name: threat.indicator.file.hash.pehash type: keyword description: "The file's pehash, if available." -- external: ecs - name: threat.indicator.provider -- external: ecs - name: labels diff --git a/packages/ti_otx/data_stream/threat/fields/agent.yml b/packages/ti_otx/data_stream/threat/fields/agent.yml index da4e652c53b..2bc58530bac 100644 --- a/packages/ti_otx/data_stream/threat/fields/agent.yml +++ b/packages/ti_otx/data_stream/threat/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/ti_otx/data_stream/threat/fields/beats.yml b/packages/ti_otx/data_stream/threat/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/ti_otx/data_stream/threat/fields/beats.yml +++ b/packages/ti_otx/data_stream/threat/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/ti_otx/data_stream/threat/fields/ecs.yml b/packages/ti_otx/data_stream/threat/fields/ecs.yml index df4f0c7661d..34fc117cd80 100644 --- a/packages/ti_otx/data_stream/threat/fields/ecs.yml +++ b/packages/ti_otx/data_stream/threat/fields/ecs.yml @@ -1,61 +1,3 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: error.message -- external: ecs - name: tags -- external: ecs - name: related.hash -- external: ecs - name: related.ip -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.created -- external: ecs - name: event.category -- external: ecs - name: event.type -- external: ecs - name: event.original -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.email.address -- external: ecs - name: threat.indicator.ip -- external: ecs - name: threat.indicator.url.domain -- external: ecs - name: threat.indicator.url.full -- external: ecs - name: threat.indicator.url.extension -- external: ecs - name: threat.indicator.url.original -- external: ecs - name: threat.indicator.url.path -- external: ecs - name: threat.indicator.url.port -- external: ecs - name: threat.indicator.url.scheme -- external: ecs - name: threat.indicator.url.query -- external: ecs - name: threat.indicator.file.type -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.file.pe.imphash - name: threat.indicator.file.hash.pehash type: keyword description: "The file's pehash, if available." -- external: ecs - name: threat.indicator.provider diff --git a/packages/ti_otx/docs/README.md b/packages/ti_otx/docs/README.md index d4989dc3553..31ce471790a 100644 --- a/packages/ti_otx/docs/README.md +++ b/packages/ti_otx/docs/README.md @@ -17,85 +17,27 @@ Retrieves all the related indicators over time, related to your pulse subscripti | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | otx.content | Extra text or descriptive content related to the indicator. | keyword | | otx.description | A description of the indicator. | keyword | | otx.id | The ID of the indicator. | keyword | | otx.indicator | The value of the indicator, for example if the type is domain, this would be the value. | keyword | | otx.title | Title describing the indicator. | keyword | | otx.type | The indicator type, can for example be "domain, email, FileHash-SHA256". | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| tags | List of keywords used to tag each event. | keyword | | threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword | | threat.feed.name | Display friendly feed name | constant_keyword | -| threat.indicator.email.address | Identifies a threat indicator as an email address (irrespective of direction). | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | | threat.indicator.file.hash.pehash | The file's pehash, if available. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.file.pe.imphash | A hash of the imports in a PE file. An imphash -- or import hash -- can be used to fingerprint binaries even after recompilation or other code-level transformations have occurred, which would change more traditional hash values. Learn more at https://www.fireeye.com/blog/threat-research/2014/01/tracking-malware-import-hashing.html. | keyword | -| threat.indicator.file.type | File type (file, dir, or symlink). | keyword | -| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| threat.indicator.url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| threat.indicator.url.full.text | Multi-field of `threat.indicator.url.full`. | match_only_text | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | -| threat.indicator.url.path | Path of the request, such as "/search". | wildcard | -| threat.indicator.url.port | Port of the request, such as 443. | long | -| threat.indicator.url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| threat.indicator.url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | An example event for `threat` looks as following: @@ -181,52 +123,15 @@ The following subscriptions are included by this API: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| labels | Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels. | object | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | otx.content | | keyword | | otx.count | | integer | | otx.created | | date | @@ -258,30 +163,9 @@ The following subscriptions are included by this API: | otx.t2 | | double | | otx.t3 | | double | | otx.title | | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| tags | List of keywords used to tag each event. | keyword | | threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword | | threat.feed.name | Display friendly feed name | constant_keyword | -| threat.indicator.email.address | Identifies a threat indicator as an email address (irrespective of direction). | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | | threat.indicator.file.hash.pehash | The file's pehash, if available. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.file.pe.imphash | A hash of the imports in a PE file. An imphash -- or import hash -- can be used to fingerprint binaries even after recompilation or other code-level transformations have occurred, which would change more traditional hash values. Learn more at https://www.fireeye.com/blog/threat-research/2014/01/tracking-malware-import-hashing.html. | keyword | -| threat.indicator.file.type | File type (file, dir, or symlink). | keyword | -| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| threat.indicator.url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| threat.indicator.url.full.text | Multi-field of `threat.indicator.url.full`. | match_only_text | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | -| threat.indicator.url.path | Path of the request, such as "/search". | wildcard | -| threat.indicator.url.port | Port of the request, such as 443. | long | -| threat.indicator.url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| threat.indicator.url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | An example event for `pulses_subscribed` looks as following: diff --git a/packages/ti_otx/manifest.yml b/packages/ti_otx/manifest.yml index 1364cc0a303..29fca1f2893 100644 --- a/packages/ti_otx/manifest.yml +++ b/packages/ti_otx/manifest.yml @@ -1,13 +1,13 @@ name: ti_otx title: AlienVault OTX -version: "1.24.1" +version: "1.25.0" description: Ingest threat intelligence indicators from AlienVault Open Threat Exchange (OTX) with Elastic Agent. type: integration format_version: "3.0.2" categories: ["security", "threat_intel"] conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" icons: - src: /img/otx.svg title: Alienvault OTX From 6bb531a1592264e28fcff9e1f5eddd902edf2219 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:28:11 +0930 Subject: [PATCH 106/121] [ti_rapid7_threat_command] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/ti_rapid7_threat_command --- .../ti_rapid7_threat_command/changelog.yml | 5 + .../data_stream/alert/fields/agent.yml | 147 ---------------- .../data_stream/alert/fields/ecs.yml | 20 --- .../data_stream/ioc/fields/agent.yml | 147 ---------------- .../data_stream/ioc/fields/ecs.yml | 103 ------------ .../vulnerability/fields/agent.yml | 147 ---------------- .../data_stream/vulnerability/fields/ecs.yml | 32 ---- .../vulnerability/fields/overridden-ecs.yml | 4 - .../ti_rapid7_threat_command/docs/README.md | 158 ------------------ .../ti_rapid7_threat_command/manifest.yml | 4 +- 10 files changed, 7 insertions(+), 760 deletions(-) delete mode 100644 packages/ti_rapid7_threat_command/data_stream/alert/fields/ecs.yml delete mode 100644 packages/ti_rapid7_threat_command/data_stream/ioc/fields/ecs.yml delete mode 100644 packages/ti_rapid7_threat_command/data_stream/vulnerability/fields/ecs.yml delete mode 100644 packages/ti_rapid7_threat_command/data_stream/vulnerability/fields/overridden-ecs.yml diff --git a/packages/ti_rapid7_threat_command/changelog.yml b/packages/ti_rapid7_threat_command/changelog.yml index 96ab632f5e6..b1620a43bf1 100644 --- a/packages/ti_rapid7_threat_command/changelog.yml +++ b/packages/ti_rapid7_threat_command/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.17.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.16.0" changes: - description: Improve handling of empty responses. diff --git a/packages/ti_rapid7_threat_command/data_stream/alert/fields/agent.yml b/packages/ti_rapid7_threat_command/data_stream/alert/fields/agent.yml index 1d37c906754..c51a2a4a1f8 100644 --- a/packages/ti_rapid7_threat_command/data_stream/alert/fields/agent.yml +++ b/packages/ti_rapid7_threat_command/data_stream/alert/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: If the host is a container. diff --git a/packages/ti_rapid7_threat_command/data_stream/alert/fields/ecs.yml b/packages/ti_rapid7_threat_command/data_stream/alert/fields/ecs.yml deleted file mode 100644 index 96396a61731..00000000000 --- a/packages/ti_rapid7_threat_command/data_stream/alert/fields/ecs.yml +++ /dev/null @@ -1,20 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.created -- external: ecs - name: event.dataset -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.module -- external: ecs - name: event.original -- external: ecs - name: event.reference -- external: ecs - name: tags diff --git a/packages/ti_rapid7_threat_command/data_stream/ioc/fields/agent.yml b/packages/ti_rapid7_threat_command/data_stream/ioc/fields/agent.yml index 1d37c906754..c51a2a4a1f8 100644 --- a/packages/ti_rapid7_threat_command/data_stream/ioc/fields/agent.yml +++ b/packages/ti_rapid7_threat_command/data_stream/ioc/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: If the host is a container. diff --git a/packages/ti_rapid7_threat_command/data_stream/ioc/fields/ecs.yml b/packages/ti_rapid7_threat_command/data_stream/ioc/fields/ecs.yml deleted file mode 100644 index efc88cc833d..00000000000 --- a/packages/ti_rapid7_threat_command/data_stream/ioc/fields/ecs.yml +++ /dev/null @@ -1,103 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.dataset -- external: ecs - name: event.kind -- external: ecs - name: event.module -- external: ecs - name: event.original -- external: ecs - name: event.risk_score -- external: ecs - name: event.type -- external: ecs - name: related.hash -- external: ecs - name: related.ip -- external: ecs - name: tags -- external: ecs - name: threat.indicator.as.number -- external: ecs - name: threat.indicator.as.organization.name -- external: ecs - name: threat.indicator.confidence -- external: ecs - name: threat.indicator.description -- external: ecs - name: threat.indicator.email.address -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.file.hash.sha384 -- external: ecs - name: threat.indicator.file.hash.sha512 -- external: ecs - name: threat.indicator.first_seen -- external: ecs - name: threat.indicator.geo.city_name -- external: ecs - name: threat.indicator.geo.continent_name -- external: ecs - name: threat.indicator.geo.country_iso_code -- external: ecs - name: threat.indicator.geo.country_name -- external: ecs - name: threat.indicator.geo.location -- external: ecs - name: threat.indicator.geo.region_iso_code -- external: ecs - name: threat.indicator.geo.region_name -- external: ecs - name: threat.indicator.ip -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.indicator.modified_at -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.url.domain -- external: ecs - name: threat.indicator.url.extension -- external: ecs - name: threat.indicator.url.fragment -- external: ecs - name: threat.indicator.url.full - ignore_above: 4096 -- external: ecs - name: threat.indicator.url.original - ignore_above: 4096 -- external: ecs - name: threat.indicator.url.password -- external: ecs - name: threat.indicator.url.path -- external: ecs - name: threat.indicator.url.port -- external: ecs - name: threat.indicator.url.query - ignore_above: 4096 -- external: ecs - name: threat.indicator.url.registered_domain -- external: ecs - name: threat.indicator.url.scheme -- external: ecs - name: threat.indicator.url.subdomain -- external: ecs - name: threat.indicator.url.top_level_domain -- external: ecs - name: threat.indicator.url.username diff --git a/packages/ti_rapid7_threat_command/data_stream/vulnerability/fields/agent.yml b/packages/ti_rapid7_threat_command/data_stream/vulnerability/fields/agent.yml index 1d37c906754..c51a2a4a1f8 100644 --- a/packages/ti_rapid7_threat_command/data_stream/vulnerability/fields/agent.yml +++ b/packages/ti_rapid7_threat_command/data_stream/vulnerability/fields/agent.yml @@ -5,162 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container ID. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host IP addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: If the host is a container. diff --git a/packages/ti_rapid7_threat_command/data_stream/vulnerability/fields/ecs.yml b/packages/ti_rapid7_threat_command/data_stream/vulnerability/fields/ecs.yml deleted file mode 100644 index e49c52a3ade..00000000000 --- a/packages/ti_rapid7_threat_command/data_stream/vulnerability/fields/ecs.yml +++ /dev/null @@ -1,32 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.dataset -- external: ecs - name: event.kind -- external: ecs - name: event.module -- external: ecs - name: event.type -- external: ecs - name: vulnerability.classification -- external: ecs - name: vulnerability.enumeration -- external: ecs - name: vulnerability.id -- external: ecs - name: vulnerability.reference -- external: ecs - name: vulnerability.scanner.vendor -- external: ecs - name: vulnerability.score.base -- external: ecs - name: vulnerability.severity -- external: ecs - name: tags diff --git a/packages/ti_rapid7_threat_command/data_stream/vulnerability/fields/overridden-ecs.yml b/packages/ti_rapid7_threat_command/data_stream/vulnerability/fields/overridden-ecs.yml deleted file mode 100644 index 230ed31e27c..00000000000 --- a/packages/ti_rapid7_threat_command/data_stream/vulnerability/fields/overridden-ecs.yml +++ /dev/null @@ -1,4 +0,0 @@ -- name: event.original - type: keyword - ignore_above: 8191 - description: Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. diff --git a/packages/ti_rapid7_threat_command/docs/README.md b/packages/ti_rapid7_threat_command/docs/README.md index fb137e15eeb..576bccbef67 100644 --- a/packages/ti_rapid7_threat_command/docs/README.md +++ b/packages/ti_rapid7_threat_command/docs/README.md @@ -334,49 +334,13 @@ An example event for `ioc` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.risk_score | Risk score or priority of the event (e.g. security solutions). Use your system's original value here. | float | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | rapid7.tc.ioc.first_seen | IOC first seen date in Unix Millisecond Timestamp. | date | @@ -397,49 +361,6 @@ An example event for `ioc` looks as following: | rapid7.tc.ioc.type | IOC type. | keyword | | rapid7.tc.ioc.value | IOC value. | keyword | | rapid7.tc.ioc.whitelisted | An indicator which states if the IOC was checked and found as whitelisted or not. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| tags | List of keywords used to tag each event. | keyword | -| threat.indicator.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| threat.indicator.as.organization.name | Organization name. | keyword | -| threat.indicator.as.organization.name.text | Multi-field of `threat.indicator.as.organization.name`. | match_only_text | -| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword | -| threat.indicator.description | Describes the type of action conducted by the threat. | keyword | -| threat.indicator.email.address | Identifies a threat indicator as an email address (irrespective of direction). | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.file.hash.sha384 | SHA384 hash. | keyword | -| threat.indicator.file.hash.sha512 | SHA512 hash. | keyword | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.geo.city_name | City name. | keyword | -| threat.indicator.geo.continent_name | Name of the continent. | keyword | -| threat.indicator.geo.country_iso_code | Country ISO code. | keyword | -| threat.indicator.geo.country_name | Country name. | keyword | -| threat.indicator.geo.location | Longitude and latitude. | geo_point | -| threat.indicator.geo.region_iso_code | Region ISO code. | keyword | -| threat.indicator.geo.region_name | Region name. | keyword | -| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| threat.indicator.url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | -| threat.indicator.url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| threat.indicator.url.full.text | Multi-field of `threat.indicator.url.full`. | match_only_text | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | -| threat.indicator.url.password | Password of the request. | keyword | -| threat.indicator.url.path | Path of the request, such as "/search". | wildcard | -| threat.indicator.url.port | Port of the request, such as 443. | long | -| threat.indicator.url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| threat.indicator.url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| threat.indicator.url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| threat.indicator.url.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| threat.indicator.url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| threat.indicator.url.username | Username of the request. | keyword | ### Alert @@ -548,48 +469,13 @@ An example event for `alert` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.reference | Reference URL linking to additional information about this event. This URL links to a static definition of this event. Alert events, indicated by `event.kind:alert`, are a common use case for this field. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | rapid7.tc.alert.assets.type | Type of an asset. | keyword | @@ -618,7 +504,6 @@ An example event for `alert` looks as following: | rapid7.tc.alert.related_threat_ids | List of related threat IDs. | keyword | | rapid7.tc.alert.takedown_status | Alert remediation status. | keyword | | rapid7.tc.alert.update_date | Last update date of an alert in Unix Millisecond Timestamp. | date | -| tags | List of keywords used to tag each event. | keyword | ### Vulnerability @@ -760,48 +645,13 @@ An example event for `vulnerability` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization ID used to identify different entities in a multi-tenant environment. Examples: AWS account ID, Google Cloud ORG ID, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container ID. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | -| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host ID. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host IP addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | | rapid7.tc.vulnerability.cpe.range.version.end.excluding | The CPE version end range. | version | @@ -833,12 +683,4 @@ An example event for `vulnerability` looks as following: | rapid7.tc.vulnerability.related.threat_actors | List of related threat actors. | keyword | | rapid7.tc.vulnerability.severity | CVE severity. Allowed values: 'Critical', 'High', 'Medium', 'Low'. | keyword | | rapid7.tc.vulnerability.update_date | CVE's update date in ISO 8601 format. | date | -| tags | List of keywords used to tag each event. | keyword | -| vulnerability.classification | The classification of the vulnerability scoring system. For example (https://www.first.org/cvss/) | keyword | -| vulnerability.enumeration | The type of identifier used for this vulnerability. For example (https://cve.mitre.org/about/) | keyword | -| vulnerability.id | The identification (ID) is the number portion of a vulnerability entry. It includes a unique identification number for the vulnerability. For example (https://cve.mitre.org/about/faqs.html#what_is_cve_id)[Common Vulnerabilities and Exposure CVE ID] | keyword | -| vulnerability.reference | A resource that provides additional information, context, and mitigations for the identified vulnerability. | keyword | -| vulnerability.scanner.vendor | The name of the vulnerability scanner vendor. | keyword | -| vulnerability.score.base | Scores can range from 0.0 to 10.0, with 10.0 being the most severe. Base scores cover an assessment for exploitability metrics (attack vector, complexity, privileges, and user interaction), impact metrics (confidentiality, integrity, and availability), and scope. For example (https://www.first.org/cvss/specification-document) | float | -| vulnerability.severity | The severity of the vulnerability can help with metrics and internal prioritization regarding remediation. For example (https://nvd.nist.gov/vuln-metrics/cvss) | keyword | diff --git a/packages/ti_rapid7_threat_command/manifest.yml b/packages/ti_rapid7_threat_command/manifest.yml index ed9246e1879..677c7b3dd73 100644 --- a/packages/ti_rapid7_threat_command/manifest.yml +++ b/packages/ti_rapid7_threat_command/manifest.yml @@ -2,13 +2,13 @@ format_version: 3.0.2 name: ti_rapid7_threat_command title: Rapid7 Threat Command # The version must be updated manually in the transform.yml files and transform APIs mentioned in README. -version: "1.16.0" +version: "1.17.0" description: Collect threat intelligence from Threat Command API with Elastic Agent. type: integration categories: ["security", "threat_intel"] conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: capabilities: - security From 405121ab453a7f6e836cdde7ca879aa6f93bc928 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:28:13 +0930 Subject: [PATCH 107/121] [ti_recordedfuture] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/ti_recordedfuture --- packages/ti_recordedfuture/changelog.yml | 5 + .../data_stream/threat/fields/agent.yml | 167 +----------------- .../data_stream/threat/fields/beats.yml | 3 - .../data_stream/threat/fields/ecs.yml | 76 -------- packages/ti_recordedfuture/docs/README.md | 68 ------- packages/ti_recordedfuture/manifest.yml | 4 +- 6 files changed, 8 insertions(+), 315 deletions(-) delete mode 100644 packages/ti_recordedfuture/data_stream/threat/fields/ecs.yml diff --git a/packages/ti_recordedfuture/changelog.yml b/packages/ti_recordedfuture/changelog.yml index 3b9e405597d..0758329930c 100644 --- a/packages/ti_recordedfuture/changelog.yml +++ b/packages/ti_recordedfuture/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.26.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.25.1" changes: - description: Adjust field mappings for transform destination index. diff --git a/packages/ti_recordedfuture/data_stream/threat/fields/agent.yml b/packages/ti_recordedfuture/data_stream/threat/fields/agent.yml index da4e652c53b..2bc58530bac 100644 --- a/packages/ti_recordedfuture/data_stream/threat/fields/agent.yml +++ b/packages/ti_recordedfuture/data_stream/threat/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/ti_recordedfuture/data_stream/threat/fields/beats.yml b/packages/ti_recordedfuture/data_stream/threat/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/ti_recordedfuture/data_stream/threat/fields/beats.yml +++ b/packages/ti_recordedfuture/data_stream/threat/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/ti_recordedfuture/data_stream/threat/fields/ecs.yml b/packages/ti_recordedfuture/data_stream/threat/fields/ecs.yml deleted file mode 100644 index 96d2e39052c..00000000000 --- a/packages/ti_recordedfuture/data_stream/threat/fields/ecs.yml +++ /dev/null @@ -1,76 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: tags -- external: ecs - name: error.message -- external: ecs - name: event.severity -- external: ecs - name: event.category -- external: ecs - name: event.ingested -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.type -- external: ecs - name: event.original -- external: ecs - name: threat.indicator.first_seen -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.ip -- external: ecs - name: threat.indicator.url.domain -- external: ecs - name: threat.indicator.url.full -- external: ecs - name: threat.indicator.url.extension -- external: ecs - name: threat.indicator.url.original -- external: ecs - name: threat.indicator.url.path -- external: ecs - name: threat.indicator.url.port -- external: ecs - name: threat.indicator.url.scheme -- external: ecs - name: threat.indicator.url.query -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.file.hash.sha512 -- external: ecs - name: threat.indicator.email.address -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.marking.tlp -- external: ecs - name: threat.indicator.confidence -- external: ecs - name: threat.indicator.as.number -- external: ecs - name: threat.indicator.as.organization.name -- external: ecs - name: threat.indicator.geo.location -- external: ecs - name: threat.indicator.geo.country_iso_code -- external: ecs - name: threat.indicator.scanner_stats -- external: ecs - name: threat.indicator.sightings -- external: ecs - name: labels diff --git a/packages/ti_recordedfuture/docs/README.md b/packages/ti_recordedfuture/docs/README.md index 99864f15a25..d2b9b5769bb 100644 --- a/packages/ti_recordedfuture/docs/README.md +++ b/packages/ti_recordedfuture/docs/README.md @@ -135,57 +135,19 @@ An example event for `threat` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| labels | Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels. | object | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | recordedfuture.evidence_details.criticality | | double | | recordedfuture.evidence_details.criticality_label | | keyword | | recordedfuture.evidence_details.evidence_string | | keyword | @@ -199,35 +161,5 @@ An example event for `threat` looks as following: | recordedfuture.list | User-configured risklist. | keyword | | recordedfuture.name | Indicator value. | keyword | | recordedfuture.risk_string | Details of risk rules observed. | keyword | -| tags | List of keywords used to tag each event. | keyword | | threat.feed.name | Display friendly feed name | constant_keyword | -| threat.indicator.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| threat.indicator.as.organization.name | Organization name. | keyword | -| threat.indicator.as.organization.name.text | Multi-field of `threat.indicator.as.organization.name`. | match_only_text | -| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword | -| threat.indicator.email.address | Identifies a threat indicator as an email address (irrespective of direction). | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.file.hash.sha512 | SHA512 hash. | keyword | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.geo.country_iso_code | Country ISO code. | keyword | -| threat.indicator.geo.location | Longitude and latitude. | geo_point | -| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.marking.tlp | Traffic Light Protocol sharing markings. | keyword | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.scanner_stats | Count of AV/EDR vendors that successfully detected malicious file or URL. | long | -| threat.indicator.sightings | Number of times this indicator was observed conducting threat activity. | long | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| threat.indicator.url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| threat.indicator.url.full.text | Multi-field of `threat.indicator.url.full`. | match_only_text | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | -| threat.indicator.url.path | Path of the request, such as "/search". | wildcard | -| threat.indicator.url.port | Port of the request, such as 443. | long | -| threat.indicator.url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| threat.indicator.url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | diff --git a/packages/ti_recordedfuture/manifest.yml b/packages/ti_recordedfuture/manifest.yml index 5b825c8e21b..1224f15c145 100644 --- a/packages/ti_recordedfuture/manifest.yml +++ b/packages/ti_recordedfuture/manifest.yml @@ -1,13 +1,13 @@ name: ti_recordedfuture title: Recorded Future -version: "1.25.1" +version: "1.26.0" description: Ingest threat intelligence indicators from Recorded Future risk lists with Elastic Agent. type: integration format_version: 3.0.2 categories: ["security", "threat_intel"] conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" screenshots: - src: /img/rf-overview.png title: "Dashboard: RecordedFuture Overview" From 354598a4cdd48f208a13603cd43806084880b693 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:28:14 +0930 Subject: [PATCH 108/121] [ti_threatconnect] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/ti_threatconnect --- packages/ti_threatconnect/_dev/build/build.yml | 1 - packages/ti_threatconnect/changelog.yml | 5 +++++ .../ti_threatconnect/data_stream/indicator/fields/beats.yml | 3 --- packages/ti_threatconnect/docs/README.md | 1 - packages/ti_threatconnect/manifest.yml | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/ti_threatconnect/_dev/build/build.yml b/packages/ti_threatconnect/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/ti_threatconnect/_dev/build/build.yml +++ b/packages/ti_threatconnect/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/ti_threatconnect/changelog.yml b/packages/ti_threatconnect/changelog.yml index 2ef77c2217b..928f5aaaac6 100644 --- a/packages/ti_threatconnect/changelog.yml +++ b/packages/ti_threatconnect/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.1.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.0.1" changes: - description: Adjust field mappings for transform destination index. diff --git a/packages/ti_threatconnect/data_stream/indicator/fields/beats.yml b/packages/ti_threatconnect/data_stream/indicator/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/ti_threatconnect/data_stream/indicator/fields/beats.yml +++ b/packages/ti_threatconnect/data_stream/indicator/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/ti_threatconnect/docs/README.md b/packages/ti_threatconnect/docs/README.md index 7ea338c3054..427b014914d 100644 --- a/packages/ti_threatconnect/docs/README.md +++ b/packages/ti_threatconnect/docs/README.md @@ -349,7 +349,6 @@ An example event for `indicator` looks as following: | input.type | Type of filebeat input. | keyword | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | | threat.feed.name | Display friendly feed name. | constant_keyword | | threat_connect.indicator.active.locked | Indicates whether the active status is locked. | boolean | | threat_connect.indicator.active.value | Indicates whether the indicator is active. | boolean | diff --git a/packages/ti_threatconnect/manifest.yml b/packages/ti_threatconnect/manifest.yml index c75b0d27c93..3afb0a38772 100644 --- a/packages/ti_threatconnect/manifest.yml +++ b/packages/ti_threatconnect/manifest.yml @@ -2,7 +2,7 @@ format_version: 3.0.3 name: ti_threatconnect title: ThreatConnect -version: 1.0.1 +version: "1.1.0" description: Collects Indicators from ThreatConnect using the Elastic Agent and saves them as logs inside Elastic type: integration categories: @@ -10,7 +10,7 @@ categories: - threat_intel conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: subscription: basic screenshots: From 810f753af0305420de14580592299f35c6352b25 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:28:15 +0930 Subject: [PATCH 109/121] [ti_threatq] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/ti_threatq --- packages/ti_threatq/changelog.yml | 5 + .../data_stream/threat/fields/agent.yml | 167 +----------------- .../data_stream/threat/fields/beats.yml | 3 - .../data_stream/threat/fields/ecs.yml | 66 ------- packages/ti_threatq/docs/README.md | 62 ------- .../transform/latest_ioc/fields/ecs.yml | 1 - packages/ti_threatq/manifest.yml | 4 +- 7 files changed, 8 insertions(+), 300 deletions(-) delete mode 100644 packages/ti_threatq/data_stream/threat/fields/ecs.yml diff --git a/packages/ti_threatq/changelog.yml b/packages/ti_threatq/changelog.yml index c0800ba0e11..c2553083e48 100644 --- a/packages/ti_threatq/changelog.yml +++ b/packages/ti_threatq/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.28.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.27.1" changes: - description: Adjust field mappings for transform destination index. diff --git a/packages/ti_threatq/data_stream/threat/fields/agent.yml b/packages/ti_threatq/data_stream/threat/fields/agent.yml index da4e652c53b..2bc58530bac 100644 --- a/packages/ti_threatq/data_stream/threat/fields/agent.yml +++ b/packages/ti_threatq/data_stream/threat/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/ti_threatq/data_stream/threat/fields/beats.yml b/packages/ti_threatq/data_stream/threat/fields/beats.yml index cb44bb29442..96190255552 100644 --- a/packages/ti_threatq/data_stream/threat/fields/beats.yml +++ b/packages/ti_threatq/data_stream/threat/fields/beats.yml @@ -7,6 +7,3 @@ - name: log.offset type: long description: Offset of the entry in the log file. -- name: log.file.path - type: keyword - description: Path to the log file. diff --git a/packages/ti_threatq/data_stream/threat/fields/ecs.yml b/packages/ti_threatq/data_stream/threat/fields/ecs.yml deleted file mode 100644 index 0c0abea1398..00000000000 --- a/packages/ti_threatq/data_stream/threat/fields/ecs.yml +++ /dev/null @@ -1,66 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: message -- external: ecs - name: tags -- external: ecs - name: error.message -- external: ecs - name: event.category -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.type -- external: ecs - name: event.created -- external: ecs - name: event.original -- name: threat.feed.name - type: keyword -- external: ecs - name: threat.indicator.first_seen -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.indicator.description -- external: ecs - name: threat.indicator.confidence -- external: ecs - name: threat.indicator.ip -- external: ecs - name: threat.indicator.url.domain -- external: ecs - name: threat.indicator.url.full -- external: ecs - name: threat.indicator.url.extension -- external: ecs - name: threat.indicator.url.original -- external: ecs - name: threat.indicator.url.path -- external: ecs - name: threat.indicator.url.port -- external: ecs - name: threat.indicator.url.scheme -- external: ecs - name: threat.indicator.url.query -- external: ecs - name: threat.indicator.email.address -- external: ecs - name: threat.indicator.provider -- external: ecs - name: threat.indicator.file.hash.md5 -- external: ecs - name: threat.indicator.file.hash.sha1 -- external: ecs - name: threat.indicator.file.hash.sha256 -- external: ecs - name: threat.indicator.file.hash.sha512 -- external: ecs - name: threat.indicator.marking.tlp -- external: ecs - name: labels diff --git a/packages/ti_threatq/docs/README.md b/packages/ti_threatq/docs/README.md index 5579da67a88..f619f9c4e67 100644 --- a/packages/ti_threatq/docs/README.md +++ b/packages/ti_threatq/docs/README.md @@ -31,82 +31,20 @@ To facilitate IOC expiration, source datastream-backed indices `.ds-logs-ti_thre | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| labels | Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels. | object | | labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | -| log.file.path | Path to the log file. | keyword | | log.flags | Flags for the log file. | keyword | | log.offset | Offset of the entry in the log file. | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | | threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword | -| threat.feed.name | | keyword | -| threat.indicator.confidence | Identifies the vendor-neutral confidence rating using the None/Low/Medium/High scale defined in Appendix A of the STIX 2.1 framework. Vendor-specific confidence scales may be added as custom fields. | keyword | -| threat.indicator.description | Describes the type of action conducted by the threat. | keyword | -| threat.indicator.email.address | Identifies a threat indicator as an email address (irrespective of direction). | keyword | -| threat.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.indicator.file.hash.sha512 | SHA512 hash. | keyword | -| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | -| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.marking.tlp | Traffic Light Protocol sharing markings. | keyword | -| threat.indicator.provider | The name of the indicator's provider. | keyword | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| threat.indicator.url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| threat.indicator.url.full.text | Multi-field of `threat.indicator.url.full`. | match_only_text | -| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | -| threat.indicator.url.path | Path of the request, such as "/search". | wildcard | -| threat.indicator.url.port | Port of the request, such as 443. | long | -| threat.indicator.url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| threat.indicator.url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | | threatq.adversaries | Adversaries that are linked to the object | keyword | | threatq.attributes | These provide additional context about an object | flattened | | threatq.created_at | Object creation time | date | diff --git a/packages/ti_threatq/elasticsearch/transform/latest_ioc/fields/ecs.yml b/packages/ti_threatq/elasticsearch/transform/latest_ioc/fields/ecs.yml index fe7f103c293..0dabd5a6594 100644 --- a/packages/ti_threatq/elasticsearch/transform/latest_ioc/fields/ecs.yml +++ b/packages/ti_threatq/elasticsearch/transform/latest_ioc/fields/ecs.yml @@ -76,4 +76,3 @@ type: constant_keyword description: Event dataset value: ti_threatq.threat - diff --git a/packages/ti_threatq/manifest.yml b/packages/ti_threatq/manifest.yml index 5d3e765d28b..f6ce83ca7a1 100644 --- a/packages/ti_threatq/manifest.yml +++ b/packages/ti_threatq/manifest.yml @@ -1,13 +1,13 @@ name: ti_threatq title: ThreatQuotient -version: "1.27.1" +version: "1.28.0" description: Ingest threat intelligence indicators from ThreatQuotient with Elastic Agent. type: integration format_version: "3.0.2" categories: ["security", "threat_intel"] conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" icons: - src: /img/threatq.svg title: ThreatQuotient From 3de5621a39dbddd4aed341e5f90608099ad59096 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:28:15 +0930 Subject: [PATCH 110/121] [ti_util] - change to kibana constraint to ^8.13.0 The conditions.kibana.version in the package manifest changed from ^8.5.0 to ^8.13.0. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/ti_util --- packages/ti_util/changelog.yml | 5 +++++ packages/ti_util/manifest.yml | 4 ++-- packages/ti_util/validation.yml | 6 +++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/ti_util/changelog.yml b/packages/ti_util/changelog.yml index 57eee3b7de4..008ca8a3909 100644 --- a/packages/ti_util/changelog.yml +++ b/packages/ti_util/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.6.0" + changes: + - description: Update the kibana constraint to ^8.13.0. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.5.0" changes: - description: Update manifest format version to v3.0.3. diff --git a/packages/ti_util/manifest.yml b/packages/ti_util/manifest.yml index b6c9189f2d9..d9bb481ce44 100644 --- a/packages/ti_util/manifest.yml +++ b/packages/ti_util/manifest.yml @@ -1,13 +1,13 @@ name: ti_util title: "Threat Intelligence Utilities" -version: "1.5.0" +version: "1.6.0" description: Prebuilt Threat Intelligence dashboard for Elastic Security categories: - security - threat_intel conditions: kibana: - version: ^8.5.0 + version: "^8.13.0" format_version: "3.0.3" type: integration screenshots: diff --git a/packages/ti_util/validation.yml b/packages/ti_util/validation.yml index 83ce7bbe929..9dcaa3b03ff 100644 --- a/packages/ti_util/validation.yml +++ b/packages/ti_util/validation.yml @@ -1,5 +1,5 @@ errors: exclude_checks: - - SVR00002 # Mandatory filters in dashboards. - - SVR00004 # References in dashboards. - - SVR00005 # Kibana version for saved tags. + - SVR00002 # Mandatory filters in dashboards. + - SVR00004 # References in dashboards. + - SVR00005 # Kibana version for saved tags. From 0b6a37e5bae5ad3993fdfde38ebbb401a874cfeb Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:28:16 +0930 Subject: [PATCH 111/121] [trellix_edr_cloud] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/trellix_edr_cloud --- packages/trellix_edr_cloud/_dev/build/build.yml | 1 - packages/trellix_edr_cloud/changelog.yml | 5 +++++ .../trellix_edr_cloud/data_stream/event/fields/beats.yml | 3 --- packages/trellix_edr_cloud/docs/README.md | 1 - packages/trellix_edr_cloud/manifest.yml | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/trellix_edr_cloud/_dev/build/build.yml b/packages/trellix_edr_cloud/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/trellix_edr_cloud/_dev/build/build.yml +++ b/packages/trellix_edr_cloud/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/trellix_edr_cloud/changelog.yml b/packages/trellix_edr_cloud/changelog.yml index b883c99f3ba..702a173c19e 100644 --- a/packages/trellix_edr_cloud/changelog.yml +++ b/packages/trellix_edr_cloud/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.2.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.1.0" changes: - description: Set sensitive values as secret. diff --git a/packages/trellix_edr_cloud/data_stream/event/fields/beats.yml b/packages/trellix_edr_cloud/data_stream/event/fields/beats.yml index 083dcfe307e..fff1b3f1b6b 100644 --- a/packages/trellix_edr_cloud/data_stream/event/fields/beats.yml +++ b/packages/trellix_edr_cloud/data_stream/event/fields/beats.yml @@ -4,9 +4,6 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. - name: aws.s3 type: group fields: diff --git a/packages/trellix_edr_cloud/docs/README.md b/packages/trellix_edr_cloud/docs/README.md index 44219233bae..bf450733c1d 100644 --- a/packages/trellix_edr_cloud/docs/README.md +++ b/packages/trellix_edr_cloud/docs/README.md @@ -357,7 +357,6 @@ An example event for `event` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | | trellix_edr_cloud.event.access_type | | keyword | | trellix_edr_cloud.event.action | | keyword | | trellix_edr_cloud.event.arguments | | keyword | diff --git a/packages/trellix_edr_cloud/manifest.yml b/packages/trellix_edr_cloud/manifest.yml index 181b095988d..9b6e613d678 100644 --- a/packages/trellix_edr_cloud/manifest.yml +++ b/packages/trellix_edr_cloud/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: trellix_edr_cloud title: Trellix EDR Cloud -version: "1.1.0" +version: "1.2.0" description: Collect logs from Trellix EDR Cloud with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - security conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: subscription: basic screenshots: From f1d94b847e9872da2c367daae9f5c61cb5d2ac2b Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:28:19 +0930 Subject: [PATCH 112/121] [trellix_epo_cloud] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/trellix_epo_cloud --- packages/trellix_epo_cloud/_dev/build/build.yml | 1 - packages/trellix_epo_cloud/changelog.yml | 5 +++++ .../trellix_epo_cloud/data_stream/device/fields/beats.yml | 3 --- .../trellix_epo_cloud/data_stream/event/fields/beats.yml | 3 --- .../trellix_epo_cloud/data_stream/group/fields/beats.yml | 3 --- packages/trellix_epo_cloud/docs/README.md | 3 --- packages/trellix_epo_cloud/manifest.yml | 4 ++-- 7 files changed, 7 insertions(+), 15 deletions(-) diff --git a/packages/trellix_epo_cloud/_dev/build/build.yml b/packages/trellix_epo_cloud/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/trellix_epo_cloud/_dev/build/build.yml +++ b/packages/trellix_epo_cloud/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/trellix_epo_cloud/changelog.yml b/packages/trellix_epo_cloud/changelog.yml index c5ab2775d1c..e7276684d5f 100644 --- a/packages/trellix_epo_cloud/changelog.yml +++ b/packages/trellix_epo_cloud/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.11.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.10.0" changes: - description: Set sensitive values as secret. diff --git a/packages/trellix_epo_cloud/data_stream/device/fields/beats.yml b/packages/trellix_epo_cloud/data_stream/device/fields/beats.yml index 80cbae91cae..cc9fcebf29b 100644 --- a/packages/trellix_epo_cloud/data_stream/device/fields/beats.yml +++ b/packages/trellix_epo_cloud/data_stream/device/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/trellix_epo_cloud/data_stream/event/fields/beats.yml b/packages/trellix_epo_cloud/data_stream/event/fields/beats.yml index 80cbae91cae..cc9fcebf29b 100644 --- a/packages/trellix_epo_cloud/data_stream/event/fields/beats.yml +++ b/packages/trellix_epo_cloud/data_stream/event/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/trellix_epo_cloud/data_stream/group/fields/beats.yml b/packages/trellix_epo_cloud/data_stream/group/fields/beats.yml index 80cbae91cae..cc9fcebf29b 100644 --- a/packages/trellix_epo_cloud/data_stream/group/fields/beats.yml +++ b/packages/trellix_epo_cloud/data_stream/group/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/trellix_epo_cloud/docs/README.md b/packages/trellix_epo_cloud/docs/README.md index 4ee6ec0b623..979c7b9e98b 100644 --- a/packages/trellix_epo_cloud/docs/README.md +++ b/packages/trellix_epo_cloud/docs/README.md @@ -215,7 +215,6 @@ An example event for `device` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | | trellix_epo_cloud.device.attributes.agent.guid | | keyword | | trellix_epo_cloud.device.attributes.agent.platform | | keyword | | trellix_epo_cloud.device.attributes.agent.state | | boolean | @@ -438,7 +437,6 @@ An example event for `event` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | | trellix_epo_cloud.event.attributes.agent.guid | | keyword | | trellix_epo_cloud.event.attributes.analyzer.dat_version | | keyword | | trellix_epo_cloud.event.attributes.analyzer.detection_method | | keyword | @@ -597,7 +595,6 @@ An example event for `group` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | | trellix_epo_cloud.group.attributes.group_type.id | | keyword | | trellix_epo_cloud.group.attributes.l1_parent.id | | keyword | | trellix_epo_cloud.group.attributes.l2_parent.id | | keyword | diff --git a/packages/trellix_epo_cloud/manifest.yml b/packages/trellix_epo_cloud/manifest.yml index 0124cb44a98..301b76ad472 100644 --- a/packages/trellix_epo_cloud/manifest.yml +++ b/packages/trellix_epo_cloud/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: trellix_epo_cloud title: Trellix ePO Cloud -version: "1.10.0" +version: "1.11.0" source: license: Elastic-2.0 description: Collect logs from Trellix ePO Cloud with Elastic Agent. @@ -10,7 +10,7 @@ categories: - security conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: subscription: basic screenshots: From 3b1f987a39532cec32d07930e6dcca801e11af11 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:28:20 +0930 Subject: [PATCH 113/121] [trendmicro] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.11.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/trendmicro --- packages/trendmicro/_dev/build/build.yml | 1 - packages/trendmicro/changelog.yml | 5 +++++ .../trendmicro/data_stream/deep_security/fields/beats.yml | 3 --- packages/trendmicro/docs/README.md | 1 - packages/trendmicro/manifest.yml | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/trendmicro/_dev/build/build.yml b/packages/trendmicro/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/trendmicro/_dev/build/build.yml +++ b/packages/trendmicro/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/trendmicro/changelog.yml b/packages/trendmicro/changelog.yml index 5dad01043b1..4d1cac06c91 100644 --- a/packages/trendmicro/changelog.yml +++ b/packages/trendmicro/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.3.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.2.0" changes: - description: Add ECS categorizations for anti-malware events. diff --git a/packages/trendmicro/data_stream/deep_security/fields/beats.yml b/packages/trendmicro/data_stream/deep_security/fields/beats.yml index 9eff736e678..9daf23f1f79 100644 --- a/packages/trendmicro/data_stream/deep_security/fields/beats.yml +++ b/packages/trendmicro/data_stream/deep_security/fields/beats.yml @@ -25,6 +25,3 @@ - name: vol type: keyword description: The serial number of the volume that contains a file. (Windows-only) -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/trendmicro/docs/README.md b/packages/trendmicro/docs/README.md index f3744df5c9a..f9b2ba2d8f5 100644 --- a/packages/trendmicro/docs/README.md +++ b/packages/trendmicro/docs/README.md @@ -163,7 +163,6 @@ An example event for `deep_security` looks as following: | log.offset | Log offset. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | | source.process.name | Source process name. | keyword | -| tags | User defined tags. | keyword | | trendmicro.deep_security.action | The action detected by the integrity rule. | keyword | | trendmicro.deep_security.aggregation_type | An integer that indicates how the event is aggregated:. | keyword | | trendmicro.deep_security.base_event_count | Base event count. | long | diff --git a/packages/trendmicro/manifest.yml b/packages/trendmicro/manifest.yml index 7ee4aef99cc..d191d1c2cb1 100644 --- a/packages/trendmicro/manifest.yml +++ b/packages/trendmicro/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.0" name: trendmicro title: Trend Micro Deep Security -version: "2.2.0" +version: "2.3.0" description: Collect logs from Trend Micro Deep Security with Elastic Agent. type: integration categories: @@ -10,7 +10,7 @@ categories: - security conditions: kibana: - version: "^8.11.0" + version: "^8.13.0" elastic: subscription: basic screenshots: From 68147fa827e91ef9b4242a6207439bf005ccdb09 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:28:23 +0930 Subject: [PATCH 114/121] [trend_micro_vision_one] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/trend_micro_vision_one --- packages/trend_micro_vision_one/changelog.yml | 5 + .../data_stream/alert/fields/agent.yml | 167 +----------------- .../data_stream/alert/fields/ecs.yml | 34 ---- .../data_stream/audit/fields/agent.yml | 167 +----------------- .../data_stream/audit/fields/ecs.yml | 20 --- .../data_stream/detection/fields/agent.yml | 167 +----------------- .../data_stream/detection/fields/ecs.yml | 90 ---------- .../trend_micro_vision_one/docs/README.md | 159 ----------------- packages/trend_micro_vision_one/manifest.yml | 4 +- 9 files changed, 10 insertions(+), 803 deletions(-) delete mode 100644 packages/trend_micro_vision_one/data_stream/alert/fields/ecs.yml delete mode 100644 packages/trend_micro_vision_one/data_stream/audit/fields/ecs.yml delete mode 100644 packages/trend_micro_vision_one/data_stream/detection/fields/ecs.yml diff --git a/packages/trend_micro_vision_one/changelog.yml b/packages/trend_micro_vision_one/changelog.yml index 4e9c309252f..977c7255282 100644 --- a/packages/trend_micro_vision_one/changelog.yml +++ b/packages/trend_micro_vision_one/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.20.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.19.1" changes: - description: Fix sample event. diff --git a/packages/trend_micro_vision_one/data_stream/alert/fields/agent.yml b/packages/trend_micro_vision_one/data_stream/alert/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/trend_micro_vision_one/data_stream/alert/fields/agent.yml +++ b/packages/trend_micro_vision_one/data_stream/alert/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/trend_micro_vision_one/data_stream/alert/fields/ecs.yml b/packages/trend_micro_vision_one/data_stream/alert/fields/ecs.yml deleted file mode 100644 index f4275f1ecc4..00000000000 --- a/packages/trend_micro_vision_one/data_stream/alert/fields/ecs.yml +++ /dev/null @@ -1,34 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.severity -- external: ecs - name: event.type -- external: ecs - name: log.level -- external: ecs - name: related.ip -- external: ecs - name: tags -- external: ecs - name: url.domain -- external: ecs - name: url.extension -- external: ecs - name: url.fragment -- external: ecs - name: url.original -- external: ecs - name: url.path -- external: ecs - name: url.scheme diff --git a/packages/trend_micro_vision_one/data_stream/audit/fields/agent.yml b/packages/trend_micro_vision_one/data_stream/audit/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/trend_micro_vision_one/data_stream/audit/fields/agent.yml +++ b/packages/trend_micro_vision_one/data_stream/audit/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/trend_micro_vision_one/data_stream/audit/fields/ecs.yml b/packages/trend_micro_vision_one/data_stream/audit/fields/ecs.yml deleted file mode 100644 index 951fd69cf1c..00000000000 --- a/packages/trend_micro_vision_one/data_stream/audit/fields/ecs.yml +++ /dev/null @@ -1,20 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: related.user -- external: ecs - name: source.user.name -- external: ecs - name: source.user.roles -- external: ecs - name: tags diff --git a/packages/trend_micro_vision_one/data_stream/detection/fields/agent.yml b/packages/trend_micro_vision_one/data_stream/detection/fields/agent.yml index e313ec82874..48f513b61aa 100644 --- a/packages/trend_micro_vision_one/data_stream/detection/fields/agent.yml +++ b/packages/trend_micro_vision_one/data_stream/detection/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/trend_micro_vision_one/data_stream/detection/fields/ecs.yml b/packages/trend_micro_vision_one/data_stream/detection/fields/ecs.yml deleted file mode 100644 index f39d1f85028..00000000000 --- a/packages/trend_micro_vision_one/data_stream/detection/fields/ecs.yml +++ /dev/null @@ -1,90 +0,0 @@ -- external: ecs - name: client.ip -- external: ecs - name: destination.domain -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.severity -- external: ecs - name: event.type -- external: ecs - name: file.hash.md5 -- external: ecs - name: file.hash.sha1 -- external: ecs - name: file.hash.sha256 -- external: ecs - name: file.name -- external: ecs - name: file.path -- external: ecs - name: file.size -- external: ecs - name: file.type -- external: ecs - name: http.request.referrer -- external: ecs - name: network.direction -- external: ecs - name: network.protocol -- external: ecs - name: observer.hostname -- external: ecs - name: observer.mac -- external: ecs - name: process.command_line -- external: ecs - name: process.name -- external: ecs - name: process.pid -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: tags -- external: ecs - name: threat.tactic.id -- external: ecs - name: url.domain -- external: ecs - name: url.original -- external: ecs - name: url.path -- external: ecs - name: url.scheme -- external: ecs - name: user.domain -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version diff --git a/packages/trend_micro_vision_one/docs/README.md b/packages/trend_micro_vision_one/docs/README.md index 444208b8605..65381dc22d2 100644 --- a/packages/trend_micro_vision_one/docs/README.md +++ b/packages/trend_micro_vision_one/docs/README.md @@ -186,54 +186,17 @@ An example event for `alert` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | -| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | | log.offset | Log offset | long | -| related.ip | All of the IPs seen on your event. | ip | -| tags | List of keywords used to tag each event. | keyword | | trend_micro_vision_one.alert.alert_provider | Alert provider. | keyword | | trend_micro_vision_one.alert.campaign | An object-ref to a campaign object. | keyword | | trend_micro_vision_one.alert.created_by | Created by. | keyword | @@ -292,13 +255,6 @@ An example event for `alert` looks as following: | trend_micro_vision_one.alert.severity | Workbench alert severity. | keyword | | trend_micro_vision_one.alert.total_indicator_count | Total indicator pattern count. | long | | trend_micro_vision_one.alert.workbench_link | Workbench URL. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | ### audit @@ -391,54 +347,17 @@ An example event for `audit` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.user.name | Short name or login of the user. | keyword | -| source.user.name.text | Multi-field of `source.user.name`. | match_only_text | -| source.user.roles | Array of user roles at the time of the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | | trend_micro_vision_one.audit.access_type | Source of the activity. | keyword | | trend_micro_vision_one.audit.activity | The activity that was performed. | keyword | | trend_micro_vision_one.audit.category | Category. | keyword | @@ -773,79 +692,17 @@ An example event for `detection` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.hash.md5 | MD5 hash. | keyword | -| file.hash.sha1 | SHA1 hash. | keyword | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| file.size | File size in bytes. Only relevant when `file.type` is "file". | long | -| file.type | File type (file, dir, or symlink). | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.referrer | Referrer for this HTTP request. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| observer.hostname | Hostname of the observer. | keyword | -| observer.mac | MAC addresses of the observer. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| process.command_line | Full command line that started the process, including the absolute path to the executable, and all arguments. Some arguments may be filtered to protect sensitive information. | wildcard | -| process.command_line.text | Multi-field of `process.command_line`. | match_only_text | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.pid | Process id. | long | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| threat.tactic.id | The id of tactic used by this threat. You can use a MITRE ATT&CK® tactic, for example. (ex. https://attack.mitre.org/tactics/TA0002/ ) | keyword | | trend_micro_vision_one.detection.action | Action by detect product. | keyword | | trend_micro_vision_one.detection.action_result | Action result by detect product. | keyword | | trend_micro_vision_one.detection.aggregated_count | Aggregated count. | long | @@ -957,20 +814,4 @@ An example event for `detection` looks as following: | trend_micro_vision_one.detection.url_cat | URL cat. | keyword | | trend_micro_vision_one.detection.user.domain | User domain. | keyword | | trend_micro_vision_one.detection.uuid | Log unique id. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | diff --git a/packages/trend_micro_vision_one/manifest.yml b/packages/trend_micro_vision_one/manifest.yml index 563662497cf..05ce42731c2 100644 --- a/packages/trend_micro_vision_one/manifest.yml +++ b/packages/trend_micro_vision_one/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: trend_micro_vision_one title: Trend Micro Vision One -version: "1.19.1" +version: "1.20.0" description: Collect logs from Trend Micro Vision One with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - edr_xdr conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" screenshots: - src: /img/trend-micro-vision-one-alert-dashboard-screenshot.png title: Trend Micro Vision One Dashboard Screenshot From dcd1f9c6ed7817f36e3410c228d73baffe741a35 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:28:26 +0930 Subject: [PATCH 115/121] [vectra_detect] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.3.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/vectra_detect --- packages/vectra_detect/_dev/build/build.yml | 1 - packages/vectra_detect/changelog.yml | 5 +++++ .../log/_dev/test/pipeline/test-common-config.yml | 2 +- .../data_stream/log/_dev/test/system/test-tcp-config.yml | 2 +- .../data_stream/log/_dev/test/system/test-tls-config.yml | 2 +- .../data_stream/log/_dev/test/system/test-udp-config.yml | 2 +- packages/vectra_detect/data_stream/log/fields/beats.yml | 3 --- packages/vectra_detect/docs/README.md | 1 - packages/vectra_detect/manifest.yml | 4 ++-- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/vectra_detect/_dev/build/build.yml b/packages/vectra_detect/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/vectra_detect/_dev/build/build.yml +++ b/packages/vectra_detect/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/vectra_detect/changelog.yml b/packages/vectra_detect/changelog.yml index c048b595383..cb6c1760da2 100644 --- a/packages/vectra_detect/changelog.yml +++ b/packages/vectra_detect/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.9.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.8.0" changes: - description: Update manifest format version to v3.0.3. diff --git a/packages/vectra_detect/data_stream/log/_dev/test/pipeline/test-common-config.yml b/packages/vectra_detect/data_stream/log/_dev/test/pipeline/test-common-config.yml index d8ab55a55b3..7695fd785a3 100644 --- a/packages/vectra_detect/data_stream/log/_dev/test/pipeline/test-common-config.yml +++ b/packages/vectra_detect/data_stream/log/_dev/test/pipeline/test-common-config.yml @@ -11,4 +11,4 @@ fields: - preserve_duplicate_custom_fields numeric_keyword_fields: - vectra_detect.log.host.groups.id - - vectra_detect.log.host.groups.triage_filters.id \ No newline at end of file + - vectra_detect.log.host.groups.triage_filters.id diff --git a/packages/vectra_detect/data_stream/log/_dev/test/system/test-tcp-config.yml b/packages/vectra_detect/data_stream/log/_dev/test/system/test-tcp-config.yml index ab9d670a404..c88ba2ed662 100644 --- a/packages/vectra_detect/data_stream/log/_dev/test/system/test-tcp-config.yml +++ b/packages/vectra_detect/data_stream/log/_dev/test/system/test-tcp-config.yml @@ -9,4 +9,4 @@ data_stream: preserve_duplicate_custom_fields: true numeric_keyword_fields: - vectra_detect.log.host.groups.id - - vectra_detect.log.host.groups.triage_filters.id \ No newline at end of file + - vectra_detect.log.host.groups.triage_filters.id diff --git a/packages/vectra_detect/data_stream/log/_dev/test/system/test-tls-config.yml b/packages/vectra_detect/data_stream/log/_dev/test/system/test-tls-config.yml index c104905f041..75abd5d905c 100644 --- a/packages/vectra_detect/data_stream/log/_dev/test/system/test-tls-config.yml +++ b/packages/vectra_detect/data_stream/log/_dev/test/system/test-tls-config.yml @@ -61,4 +61,4 @@ data_stream: preserve_duplicate_custom_fields: true numeric_keyword_fields: - vectra_detect.log.host.groups.id - - vectra_detect.log.host.groups.triage_filters.id \ No newline at end of file + - vectra_detect.log.host.groups.triage_filters.id diff --git a/packages/vectra_detect/data_stream/log/_dev/test/system/test-udp-config.yml b/packages/vectra_detect/data_stream/log/_dev/test/system/test-udp-config.yml index 769f35216e9..a52ed7dc9f3 100644 --- a/packages/vectra_detect/data_stream/log/_dev/test/system/test-udp-config.yml +++ b/packages/vectra_detect/data_stream/log/_dev/test/system/test-udp-config.yml @@ -9,4 +9,4 @@ data_stream: preserve_duplicate_custom_fields: true numeric_keyword_fields: - vectra_detect.log.host.groups.id - - vectra_detect.log.host.groups.triage_filters.id \ No newline at end of file + - vectra_detect.log.host.groups.triage_filters.id diff --git a/packages/vectra_detect/data_stream/log/fields/beats.yml b/packages/vectra_detect/data_stream/log/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/vectra_detect/data_stream/log/fields/beats.yml +++ b/packages/vectra_detect/data_stream/log/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/vectra_detect/docs/README.md b/packages/vectra_detect/docs/README.md index 99975b4ff10..a763f59e963 100644 --- a/packages/vectra_detect/docs/README.md +++ b/packages/vectra_detect/docs/README.md @@ -196,7 +196,6 @@ An example event for `log` looks as following: | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | User defined tags. | keyword | | vectra_detect.log.account.access_history.id | | keyword | | vectra_detect.log.account.access_history.last_seen | | date | | vectra_detect.log.account.access_history.privilege_category | | keyword | diff --git a/packages/vectra_detect/manifest.yml b/packages/vectra_detect/manifest.yml index 22cc96ebfea..8750bb58355 100644 --- a/packages/vectra_detect/manifest.yml +++ b/packages/vectra_detect/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: vectra_detect title: Vectra Detect -version: "1.8.0" +version: "1.9.0" source: license: Elastic-2.0 description: Collect logs from Vectra Detect with Elastic Agent. @@ -9,7 +9,7 @@ type: integration categories: ["security", "network_security"] conditions: kibana: - version: ^8.3.0 + version: "^8.13.0" elastic: subscription: basic screenshots: From cf76144c7378b34aa0704fd68f82168e354d82a0 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:28:29 +0930 Subject: [PATCH 116/121] [wiz] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/wiz --- packages/wiz/_dev/build/build.yml | 1 - packages/wiz/changelog.yml | 5 +++++ packages/wiz/data_stream/audit/fields/beats.yml | 3 --- .../issue/_dev/test/pipeline/test-issue.log-expected.json | 4 ++-- packages/wiz/data_stream/issue/fields/beats.yml | 3 --- .../_dev/test/pipeline/test-vulnerability.log-expected.json | 6 +++--- packages/wiz/data_stream/vulnerability/fields/beats.yml | 3 --- packages/wiz/docs/README.md | 3 --- packages/wiz/manifest.yml | 4 ++-- 9 files changed, 12 insertions(+), 20 deletions(-) diff --git a/packages/wiz/_dev/build/build.yml b/packages/wiz/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/wiz/_dev/build/build.yml +++ b/packages/wiz/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/wiz/changelog.yml b/packages/wiz/changelog.yml index e2e3eb5625d..8f0947f72ab 100644 --- a/packages/wiz/changelog.yml +++ b/packages/wiz/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.2.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.1.1" changes: - description: Add cloudsecurity_cdr sub category label diff --git a/packages/wiz/data_stream/audit/fields/beats.yml b/packages/wiz/data_stream/audit/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/wiz/data_stream/audit/fields/beats.yml +++ b/packages/wiz/data_stream/audit/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/wiz/data_stream/issue/_dev/test/pipeline/test-issue.log-expected.json b/packages/wiz/data_stream/issue/_dev/test/pipeline/test-issue.log-expected.json index 03e8eaaf207..b7c650d39c9 100644 --- a/packages/wiz/data_stream/issue/_dev/test/pipeline/test-issue.log-expected.json +++ b/packages/wiz/data_stream/issue/_dev/test/pipeline/test-issue.log-expected.json @@ -16,7 +16,7 @@ "created": "2023-08-23T07:56:09.903Z", "id": "fff9cffd-64a7-412c-9535-cf837f4b0b40", "kind": "event", - "original": "{\"createdAt\": \"2023-08-23T07:56:09.903743Z\",\"dueAt\": \"2023-08-30T21:00:00Z\",\"entitySnapshot\": {\"cloudPlatform\": \"Kubernetes\",\"cloudProviderURL\": \"https://portal.az.com/#@sectest.on.com/resource//subscriptions/\",\"externalId\": \"k8s/clusterrole/aaa8e7ca2bf9bc85a75d5bbdd8ffd08d69f8852782a6341c3c3519sad45/system:aggregate-to-edit/12\",\"id\": \"e507d472-b7da-5f05-9b25-72a271336b14\",\"name\": \"system:aggregate-to-edit\",\"nativeType\": \"ClusterRole\",\"providerId\": \"k8s/clusterrole/aaa8e7ca2bf9bc85a75d5bbdd8ffd08d69f8852782a6341c3c3519bac0f24ae9/system:aggregate-to-edit/12\",\"region\": \"us-01\",\"resourceGroupExternalId\": \"/subscriptions/cfd132be-3bc7-4f86-8efd-ed53ae498fec/resourcegroups/test-selfmanaged-eastus\",\"status\": \"Active\",\"subscriptionExternalId\": \"998231069301\",\"subscriptionName\": \"demo-integrations\",\"subscriptionTags\": {},\"tags\": {\"kubernetes.io/bootstrapping\": \"rbac-defaults\",\"rbac.authorization.k8s.io/aggregate-to-edit\": \"true\"},\"type\": \"ACCESS_ROLE\"},\"id\": \"fff9cffd-64a7-412c-9535-cf837f4b0b40\",\"notes\": [{\"createdAt\": \"2023-08-23T07:56:09.903743Z\",\"serviceAccount\": {\"name\": \"rev-ke\"},\"text\": \"updated\",\"updatedAt\": \"2023-08-09T23:10:22.588721Z\",\"user\":{\"name\":\"admin\",\"email\":\"admin@example.com\"}},{\"createdAt\": \"2023-08-09T23:08:49.918941Z\",\"serviceAccount\": {\"name\": \"rev-ke2\"},\"text\": \"updated\",\"updatedAt\": \"2023-08-09T23:10:22.591487Z\",\"user\":{\"name\":\"root\",\"email\":\"root@example.com\"}}],\"projects\": [{\"businessUnit\": \"\",\"id\": \"83b76efe-a7b6-5762-8a53-8e8f59e68bd8\",\"name\": \"Project 2\",\"riskProfile\": {\"businessImpact\": \"MBI\"},\"slug\": \"project-2\"},{\"businessUnit\": \"Dev\",\"id\": \"af52828c-4eb1-5c4e-847c-ebc3a5ead531\",\"name\": \"project 4\",\"riskProfile\": {\"businessImpact\": \"MBI\"},\"slug\": \"project-4\"},{\"businessUnit\": \"Dev\",\"id\": \"d6ac50bb-aec0-52fc-80ab-bacd7b02f178\",\"name\": \"Project1\",\"riskProfile\": {\"businessImpact\": \"MBI\"},\"slug\": \"project1\"}],\"resolvedAt\": \"2023-08-09T23:10:22.588721Z\",\"serviceTickets\": [{\"externalId\": \"638361121bbfdd10f6c1cbf3604bcb7e\",\"name\": \"SIR0010002\",\"url\": \"https://ven05658.testing.com/nav_to.do?uri=%2Fsn_si_incident.do%3Fsys_id%3D6385248sdsae421\"}],\"severity\": \"INFORMATIONAL\",\"sourceRule\": {\"__typename\": \"Control\",\"controlDescription\": \"These EKS principals assume roles that provide bind, escalate and impersonate permissions. \\n\\nThe `bind` permission allows users to create bindings to roles with rights they do not already have. The `escalate` permission allows users effectively escalate their privileges. The `impersonate` permission allows users to impersonate and gain the rights of other users in the cluster. Running containers with these permissions has the potential to effectively allow privilege escalation to the cluster-admin level.\",\"id\": \"wc-id-1335\",\"name\": \"EKS principals assume roles that provide bind, escalate and impersonate permissions\",\"resolutionRecommendation\": \"To follow the principle of least privilege and minimize the risk of unauthorized access and data breaches, it is recommended not to grant `bind`, `escalate` or `impersonate` permissions.\",\"securitySubCategories\": [{\"category\": {\"framework\": {\"name\": \"CIS EKS 1.2.0\"},\"name\": \"4.1 RBAC and Service Accounts\"},\"title\": \"4.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster - Level 1 (Manual)\"},{\"category\": {\"framework\": {\"name\": \"Wiz for Risk Assessment\"},\"name\": \"Identity Management\"},\"title\": \"Privileged principal\"},{\"category\": {\"framework\": {\"name\": \"Wiz\"},\"name\": \"9 Container Security\"},\"title\": \"Container Security\"},{\"category\": {\"framework\": {\"name\": \"Wiz for Risk Assessment\"},\"name\": \"Container \u0026 Kubernetes Security\"},\"title\": \"Cluster misconfiguration\"}]},\"status\": \"IN_PROGRESS\",\"statusChangedAt\": \"2023-07-31T06:26:08.708199Z\",\"updatedAt\": \"2023-08-14T06:06:18.331647Z\"}", + "original": "{\"createdAt\": \"2023-08-23T07:56:09.903743Z\",\"dueAt\": \"2023-08-30T21:00:00Z\",\"entitySnapshot\": {\"cloudPlatform\": \"Kubernetes\",\"cloudProviderURL\": \"https://portal.az.com/#@sectest.on.com/resource//subscriptions/\",\"externalId\": \"k8s/clusterrole/aaa8e7ca2bf9bc85a75d5bbdd8ffd08d69f8852782a6341c3c3519sad45/system:aggregate-to-edit/12\",\"id\": \"e507d472-b7da-5f05-9b25-72a271336b14\",\"name\": \"system:aggregate-to-edit\",\"nativeType\": \"ClusterRole\",\"providerId\": \"k8s/clusterrole/aaa8e7ca2bf9bc85a75d5bbdd8ffd08d69f8852782a6341c3c3519bac0f24ae9/system:aggregate-to-edit/12\",\"region\": \"us-01\",\"resourceGroupExternalId\": \"/subscriptions/cfd132be-3bc7-4f86-8efd-ed53ae498fec/resourcegroups/test-selfmanaged-eastus\",\"status\": \"Active\",\"subscriptionExternalId\": \"998231069301\",\"subscriptionName\": \"demo-integrations\",\"subscriptionTags\": {},\"tags\": {\"kubernetes.io/bootstrapping\": \"rbac-defaults\",\"rbac.authorization.k8s.io/aggregate-to-edit\": \"true\"},\"type\": \"ACCESS_ROLE\"},\"id\": \"fff9cffd-64a7-412c-9535-cf837f4b0b40\",\"notes\": [{\"createdAt\": \"2023-08-23T07:56:09.903743Z\",\"serviceAccount\": {\"name\": \"rev-ke\"},\"text\": \"updated\",\"updatedAt\": \"2023-08-09T23:10:22.588721Z\",\"user\":{\"name\":\"admin\",\"email\":\"admin@example.com\"}},{\"createdAt\": \"2023-08-09T23:08:49.918941Z\",\"serviceAccount\": {\"name\": \"rev-ke2\"},\"text\": \"updated\",\"updatedAt\": \"2023-08-09T23:10:22.591487Z\",\"user\":{\"name\":\"root\",\"email\":\"root@example.com\"}}],\"projects\": [{\"businessUnit\": \"\",\"id\": \"83b76efe-a7b6-5762-8a53-8e8f59e68bd8\",\"name\": \"Project 2\",\"riskProfile\": {\"businessImpact\": \"MBI\"},\"slug\": \"project-2\"},{\"businessUnit\": \"Dev\",\"id\": \"af52828c-4eb1-5c4e-847c-ebc3a5ead531\",\"name\": \"project 4\",\"riskProfile\": {\"businessImpact\": \"MBI\"},\"slug\": \"project-4\"},{\"businessUnit\": \"Dev\",\"id\": \"d6ac50bb-aec0-52fc-80ab-bacd7b02f178\",\"name\": \"Project1\",\"riskProfile\": {\"businessImpact\": \"MBI\"},\"slug\": \"project1\"}],\"resolvedAt\": \"2023-08-09T23:10:22.588721Z\",\"serviceTickets\": [{\"externalId\": \"638361121bbfdd10f6c1cbf3604bcb7e\",\"name\": \"SIR0010002\",\"url\": \"https://ven05658.testing.com/nav_to.do?uri=%2Fsn_si_incident.do%3Fsys_id%3D6385248sdsae421\"}],\"severity\": \"INFORMATIONAL\",\"sourceRule\": {\"__typename\": \"Control\",\"controlDescription\": \"These EKS principals assume roles that provide bind, escalate and impersonate permissions. \\n\\nThe `bind` permission allows users to create bindings to roles with rights they do not already have. The `escalate` permission allows users effectively escalate their privileges. The `impersonate` permission allows users to impersonate and gain the rights of other users in the cluster. Running containers with these permissions has the potential to effectively allow privilege escalation to the cluster-admin level.\",\"id\": \"wc-id-1335\",\"name\": \"EKS principals assume roles that provide bind, escalate and impersonate permissions\",\"resolutionRecommendation\": \"To follow the principle of least privilege and minimize the risk of unauthorized access and data breaches, it is recommended not to grant `bind`, `escalate` or `impersonate` permissions.\",\"securitySubCategories\": [{\"category\": {\"framework\": {\"name\": \"CIS EKS 1.2.0\"},\"name\": \"4.1 RBAC and Service Accounts\"},\"title\": \"4.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster - Level 1 (Manual)\"},{\"category\": {\"framework\": {\"name\": \"Wiz for Risk Assessment\"},\"name\": \"Identity Management\"},\"title\": \"Privileged principal\"},{\"category\": {\"framework\": {\"name\": \"Wiz\"},\"name\": \"9 Container Security\"},\"title\": \"Container Security\"},{\"category\": {\"framework\": {\"name\": \"Wiz for Risk Assessment\"},\"name\": \"Container & Kubernetes Security\"},\"title\": \"Cluster misconfiguration\"}]},\"status\": \"IN_PROGRESS\",\"statusChangedAt\": \"2023-07-31T06:26:08.708199Z\",\"updatedAt\": \"2023-08-14T06:06:18.331647Z\"}", "type": [ "info" ] @@ -171,7 +171,7 @@ "framework": { "name": "Wiz for Risk Assessment" }, - "name": "Container \u0026 Kubernetes Security" + "name": "Container & Kubernetes Security" }, "title": "Cluster misconfiguration" } diff --git a/packages/wiz/data_stream/issue/fields/beats.yml b/packages/wiz/data_stream/issue/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/wiz/data_stream/issue/fields/beats.yml +++ b/packages/wiz/data_stream/issue/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/wiz/data_stream/vulnerability/_dev/test/pipeline/test-vulnerability.log-expected.json b/packages/wiz/data_stream/vulnerability/_dev/test/pipeline/test-vulnerability.log-expected.json index df5e3717adc..573daea31d4 100644 --- a/packages/wiz/data_stream/vulnerability/_dev/test/pipeline/test-vulnerability.log-expected.json +++ b/packages/wiz/data_stream/vulnerability/_dev/test/pipeline/test-vulnerability.log-expected.json @@ -17,7 +17,7 @@ "vulnerability" ], "kind": "alert", - "original": "{\"id\":\"5e95ff50-5490-514e-87f7-11e56f3230ff\",\"portalUrl\":\"https://app.wiz.io/explorer/vulnerability-findings#~(entity~(~'xxx-xxx*2cSECURITY_TOOL_FINDING))\",\"name\":\"CVE-2020-3333\",\"CVEDescription\":\"In LibTIFF, there is a memory malloc failure in tif_pixarlog.c. A crafted TIFF document can lead to an abort, resulting in a remote denial of service attack.\",\"CVSSSeverity\":\"MEDIUM\",\"score\":5.5,\"exploitabilityScore\":1.8,\"impactScore\":3.6,\"dataSourceName\":\"data Source\",\"hasExploit\":false,\"hasCisaKevExploit\":false,\"status\":\"OPEN\",\"vendorSeverity\":\"MEDIUM\",\"firstDetectedAt\":\"2022-05-01T11:36:10.063767Z\",\"lastDetectedAt\":\"2023-08-16T18:40:57Z\",\"resolvedAt\":\"2023-08-16T18:40:57Z\",\"description\":\"Thepackage`libtiff`version`4.0.3-35.amzn2`wasdetectedin`YUMpackagemanager`onamachinerunning`Amazon2(Karoo)`isvulnerableto`CVE-2020-35522`,whichexistsinversions`\u003c4.0.3-35.amzn2.0.1`.\\n\\nThevulnerabilitywasfoundinthe[OfficialAmazonLinuxSecurityAdvisories](https://alas.aws.amazon.com/AL2/ALAS-2022-1780.html)withvendorseverity:`Medium`([NVD](https://nvd.nist.gov/vuln/detail/CVE-2020-35522)severity:`Medium`).\\n\\nThevulnerabilitycanberemediatedbyupdatingthepackagetoversion`4.0.3-35.amzn2.0.1`orhigher,using`yumupdatelibtiff`.\",\"remediation\":\"yumupdatelibtiff\",\"detailedName\":\"libtiff\",\"version\":\"4.0.3-35.amzn2\",\"fixedVersion\":\"4.0.3-35.amzn2.0.1\",\"detectionMethod\":\"PACKAGE\",\"link\":\"https://alas.aws.amazon.com/AL2/ALAS-2022-1780.html\",\"locationPath\":\"package/library/file\",\"resolutionReason\":\"resolutionReason\",\"epssSeverity\":\"LOW\",\"epssPercentile\":46.2,\"epssProbability\":0.1,\"validatedInRuntime\":true,\"layerMetadata\":{\"id\":\"5e95ff50-5490-514e-87f7-11e56f3230ff\",\"details\":\"xxxx\",\"isBaseLayer\":true},\"projects\":[{\"id\":\"83b76efe-a7b6-5762-8a53-8e8f59e68bd8\",\"name\":\"Project2\",\"slug\":\"project-2\",\"businessUnit\":\"\",\"riskProfile\":{\"businessImpact\":\"MBI\"}},{\"id\":\"af52828c-4eb1-5c4e-847c-ebc3a5ead531\",\"name\":\"project4\",\"slug\":\"project-4\",\"businessUnit\":\"Dev\",\"riskProfile\":{\"businessImpact\":\"MBI\"}},{\"id\":\"d6ac50bb-aec0-52fc-80ab-bacd7b02f178\",\"name\":\"Project1\",\"slug\":\"project1\",\"businessUnit\":\"Dev\",\"riskProfile\":{\"businessImpact\":\"MBI\"}}],\"ignoreRules\":{\"enabled\":true,\"expiredAt\":\"2023-08-16T18:40:57Z\",\"id\":\"aj3jqtvnaf\",\"name\":\"abc\"},\"vulnerableAsset\":{\"id\":\"c828de0d-4c42-5b1c-946b-2edee094d0b3\",\"type\":\"VIRTUAL_MACHINE\",\"name\":\"test-4\",\"region\":\"us-east-1\",\"providerUniqueId\":\"arn:aws:ec2:us-east-1:998231069301:instance/i-0a0f7e1451da5f4a3\",\"cloudProviderURL\":\"https://us-east-1.console.aws.amazon.com/ec2/v2/home?region=us-east-1#InstanceDetails:instanceId=i-0a0f7e1451da5f4a3\",\"cloudPlatform\":\"AWS\",\"status\":\"Active\",\"subscriptionName\":\"wiz-integrations\",\"subscriptionExternalId\":\"998231069301\",\"subscriptionId\":\"94e76baa-85fd-5928-b829-1669a2ca9660\",\"tags\":{\"Name\":\"test-4\"},\"hasLimitedInternetExposure\":true,\"hasWideInternetExposure\":true,\"isAccessibleFromVPN\":false,\"isAccessibleFromOtherVnets\":false,\"isAccessibleFromOtherSubscriptions\":false,\"operatingSystem\":\"Linux\",\"ipAddresses\":[\"89.160.20.112\",\"89.160.20.128\"]}}", + "original": "{\"id\":\"5e95ff50-5490-514e-87f7-11e56f3230ff\",\"portalUrl\":\"https://app.wiz.io/explorer/vulnerability-findings#~(entity~(~'xxx-xxx*2cSECURITY_TOOL_FINDING))\",\"name\":\"CVE-2020-3333\",\"CVEDescription\":\"In LibTIFF, there is a memory malloc failure in tif_pixarlog.c. A crafted TIFF document can lead to an abort, resulting in a remote denial of service attack.\",\"CVSSSeverity\":\"MEDIUM\",\"score\":5.5,\"exploitabilityScore\":1.8,\"impactScore\":3.6,\"dataSourceName\":\"data Source\",\"hasExploit\":false,\"hasCisaKevExploit\":false,\"status\":\"OPEN\",\"vendorSeverity\":\"MEDIUM\",\"firstDetectedAt\":\"2022-05-01T11:36:10.063767Z\",\"lastDetectedAt\":\"2023-08-16T18:40:57Z\",\"resolvedAt\":\"2023-08-16T18:40:57Z\",\"description\":\"Thepackage`libtiff`version`4.0.3-35.amzn2`wasdetectedin`YUMpackagemanager`onamachinerunning`Amazon2(Karoo)`isvulnerableto`CVE-2020-35522`,whichexistsinversions`<4.0.3-35.amzn2.0.1`.\\n\\nThevulnerabilitywasfoundinthe[OfficialAmazonLinuxSecurityAdvisories](https://alas.aws.amazon.com/AL2/ALAS-2022-1780.html)withvendorseverity:`Medium`([NVD](https://nvd.nist.gov/vuln/detail/CVE-2020-35522)severity:`Medium`).\\n\\nThevulnerabilitycanberemediatedbyupdatingthepackagetoversion`4.0.3-35.amzn2.0.1`orhigher,using`yumupdatelibtiff`.\",\"remediation\":\"yumupdatelibtiff\",\"detailedName\":\"libtiff\",\"version\":\"4.0.3-35.amzn2\",\"fixedVersion\":\"4.0.3-35.amzn2.0.1\",\"detectionMethod\":\"PACKAGE\",\"link\":\"https://alas.aws.amazon.com/AL2/ALAS-2022-1780.html\",\"locationPath\":\"package/library/file\",\"resolutionReason\":\"resolutionReason\",\"epssSeverity\":\"LOW\",\"epssPercentile\":46.2,\"epssProbability\":0.1,\"validatedInRuntime\":true,\"layerMetadata\":{\"id\":\"5e95ff50-5490-514e-87f7-11e56f3230ff\",\"details\":\"xxxx\",\"isBaseLayer\":true},\"projects\":[{\"id\":\"83b76efe-a7b6-5762-8a53-8e8f59e68bd8\",\"name\":\"Project2\",\"slug\":\"project-2\",\"businessUnit\":\"\",\"riskProfile\":{\"businessImpact\":\"MBI\"}},{\"id\":\"af52828c-4eb1-5c4e-847c-ebc3a5ead531\",\"name\":\"project4\",\"slug\":\"project-4\",\"businessUnit\":\"Dev\",\"riskProfile\":{\"businessImpact\":\"MBI\"}},{\"id\":\"d6ac50bb-aec0-52fc-80ab-bacd7b02f178\",\"name\":\"Project1\",\"slug\":\"project1\",\"businessUnit\":\"Dev\",\"riskProfile\":{\"businessImpact\":\"MBI\"}}],\"ignoreRules\":{\"enabled\":true,\"expiredAt\":\"2023-08-16T18:40:57Z\",\"id\":\"aj3jqtvnaf\",\"name\":\"abc\"},\"vulnerableAsset\":{\"id\":\"c828de0d-4c42-5b1c-946b-2edee094d0b3\",\"type\":\"VIRTUAL_MACHINE\",\"name\":\"test-4\",\"region\":\"us-east-1\",\"providerUniqueId\":\"arn:aws:ec2:us-east-1:998231069301:instance/i-0a0f7e1451da5f4a3\",\"cloudProviderURL\":\"https://us-east-1.console.aws.amazon.com/ec2/v2/home?region=us-east-1#InstanceDetails:instanceId=i-0a0f7e1451da5f4a3\",\"cloudPlatform\":\"AWS\",\"status\":\"Active\",\"subscriptionName\":\"wiz-integrations\",\"subscriptionExternalId\":\"998231069301\",\"subscriptionId\":\"94e76baa-85fd-5928-b829-1669a2ca9660\",\"tags\":{\"Name\":\"test-4\"},\"hasLimitedInternetExposure\":true,\"hasWideInternetExposure\":true,\"isAccessibleFromVPN\":false,\"isAccessibleFromOtherVnets\":false,\"isAccessibleFromOtherSubscriptions\":false,\"operatingSystem\":\"Linux\",\"ipAddresses\":[\"89.160.20.112\",\"89.160.20.128\"]}}", "type": [ "info" ] @@ -27,7 +27,7 @@ "family": "Linux" } }, - "message": "Thepackage`libtiff`version`4.0.3-35.amzn2`wasdetectedin`YUMpackagemanager`onamachinerunning`Amazon2(Karoo)`isvulnerableto`CVE-2020-35522`,whichexistsinversions`\u003c4.0.3-35.amzn2.0.1`.\n\nThevulnerabilitywasfoundinthe[OfficialAmazonLinuxSecurityAdvisories](https://alas.aws.amazon.com/AL2/ALAS-2022-1780.html)withvendorseverity:`Medium`([NVD](https://nvd.nist.gov/vuln/detail/CVE-2020-35522)severity:`Medium`).\n\nThevulnerabilitycanberemediatedbyupdatingthepackagetoversion`4.0.3-35.amzn2.0.1`orhigher,using`yumupdatelibtiff`.", + "message": "Thepackage`libtiff`version`4.0.3-35.amzn2`wasdetectedin`YUMpackagemanager`onamachinerunning`Amazon2(Karoo)`isvulnerableto`CVE-2020-35522`,whichexistsinversions`<4.0.3-35.amzn2.0.1`.\n\nThevulnerabilitywasfoundinthe[OfficialAmazonLinuxSecurityAdvisories](https://alas.aws.amazon.com/AL2/ALAS-2022-1780.html)withvendorseverity:`Medium`([NVD](https://nvd.nist.gov/vuln/detail/CVE-2020-35522)severity:`Medium`).\n\nThevulnerabilitycanberemediatedbyupdatingthepackagetoversion`4.0.3-35.amzn2.0.1`orhigher,using`yumupdatelibtiff`.", "related": { "ip": [ "89.160.20.112", @@ -49,7 +49,7 @@ "cve_description": "In LibTIFF, there is a memory malloc failure in tif_pixarlog.c. A crafted TIFF document can lead to an abort, resulting in a remote denial of service attack.", "cvss_severity": "MEDIUM", "data_source_name": "data Source", - "description": "Thepackage`libtiff`version`4.0.3-35.amzn2`wasdetectedin`YUMpackagemanager`onamachinerunning`Amazon2(Karoo)`isvulnerableto`CVE-2020-35522`,whichexistsinversions`\u003c4.0.3-35.amzn2.0.1`.\n\nThevulnerabilitywasfoundinthe[OfficialAmazonLinuxSecurityAdvisories](https://alas.aws.amazon.com/AL2/ALAS-2022-1780.html)withvendorseverity:`Medium`([NVD](https://nvd.nist.gov/vuln/detail/CVE-2020-35522)severity:`Medium`).\n\nThevulnerabilitycanberemediatedbyupdatingthepackagetoversion`4.0.3-35.amzn2.0.1`orhigher,using`yumupdatelibtiff`.", + "description": "Thepackage`libtiff`version`4.0.3-35.amzn2`wasdetectedin`YUMpackagemanager`onamachinerunning`Amazon2(Karoo)`isvulnerableto`CVE-2020-35522`,whichexistsinversions`<4.0.3-35.amzn2.0.1`.\n\nThevulnerabilitywasfoundinthe[OfficialAmazonLinuxSecurityAdvisories](https://alas.aws.amazon.com/AL2/ALAS-2022-1780.html)withvendorseverity:`Medium`([NVD](https://nvd.nist.gov/vuln/detail/CVE-2020-35522)severity:`Medium`).\n\nThevulnerabilitycanberemediatedbyupdatingthepackagetoversion`4.0.3-35.amzn2.0.1`orhigher,using`yumupdatelibtiff`.", "detailed_name": "libtiff", "detection_method": "PACKAGE", "epss": { diff --git a/packages/wiz/data_stream/vulnerability/fields/beats.yml b/packages/wiz/data_stream/vulnerability/fields/beats.yml index b3701b581cf..4084f1dc7f5 100644 --- a/packages/wiz/data_stream/vulnerability/fields/beats.yml +++ b/packages/wiz/data_stream/vulnerability/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/wiz/docs/README.md b/packages/wiz/docs/README.md index 939f816e4f5..7b62a146861 100644 --- a/packages/wiz/docs/README.md +++ b/packages/wiz/docs/README.md @@ -190,7 +190,6 @@ An example event for `audit` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | | wiz.audit.action | | keyword | | wiz.audit.action_parameters.client_id | | keyword | | wiz.audit.action_parameters.groups | | flattened | @@ -430,7 +429,6 @@ An example event for `issue` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | | wiz.issue.created_at | | date | | wiz.issue.due_at | | date | | wiz.issue.entity_snapshot.cloud.platform | | keyword | @@ -677,7 +675,6 @@ An example event for `vulnerability` looks as following: | event.module | Event module. | constant_keyword | | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | -| tags | User defined tags. | keyword | | wiz.vulnerability.cve_description | | keyword | | wiz.vulnerability.cvss_severity | | keyword | | wiz.vulnerability.data_source_name | | keyword | diff --git a/packages/wiz/manifest.yml b/packages/wiz/manifest.yml index b57f47aef86..717fe566e53 100644 --- a/packages/wiz/manifest.yml +++ b/packages/wiz/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.2 name: wiz title: Wiz -version: "1.1.1" +version: "1.2.0" description: Collect logs from Wiz with Elastic Agent. type: integration categories: @@ -9,7 +9,7 @@ categories: - cloudsecurity_cdr conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" elastic: subscription: "basic" screenshots: From aa97408681b33197dc28be1445d72791224d2f18 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:28:30 +0930 Subject: [PATCH 117/121] [zerofox] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/zerofox --- packages/zerofox/changelog.yml | 5 + .../data_stream/alerts/fields/agent.yml | 167 +----------------- .../data_stream/alerts/fields/base-fields.yml | 5 - .../zerofox/data_stream/alerts/fields/ecs.yml | 32 ---- packages/zerofox/docs/README.md | 44 ----- packages/zerofox/manifest.yml | 4 +- 6 files changed, 8 insertions(+), 249 deletions(-) delete mode 100644 packages/zerofox/data_stream/alerts/fields/ecs.yml diff --git a/packages/zerofox/changelog.yml b/packages/zerofox/changelog.yml index 61053e7f158..fe6a1f33ac9 100644 --- a/packages/zerofox/changelog.yml +++ b/packages/zerofox/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.25.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.24.0" changes: - description: Improve handling of empty responses. diff --git a/packages/zerofox/data_stream/alerts/fields/agent.yml b/packages/zerofox/data_stream/alerts/fields/agent.yml index da4e652c53b..2bc58530bac 100644 --- a/packages/zerofox/data_stream/alerts/fields/agent.yml +++ b/packages/zerofox/data_stream/alerts/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/zerofox/data_stream/alerts/fields/base-fields.yml b/packages/zerofox/data_stream/alerts/fields/base-fields.yml index 0e4b6bde4fd..9d1a033605b 100644 --- a/packages/zerofox/data_stream/alerts/fields/base-fields.yml +++ b/packages/zerofox/data_stream/alerts/fields/base-fields.yml @@ -27,8 +27,3 @@ type: constant_keyword description: Event dataset value: zerofox.alerts -- name: tags - description: List of keywords used to tag each event. - example: '["production", "env2"]' - ignore_above: 1024 - type: keyword diff --git a/packages/zerofox/data_stream/alerts/fields/ecs.yml b/packages/zerofox/data_stream/alerts/fields/ecs.yml deleted file mode 100644 index ebc4e721e65..00000000000 --- a/packages/zerofox/data_stream/alerts/fields/ecs.yml +++ /dev/null @@ -1,32 +0,0 @@ -- name: ecs.version - external: ecs -- name: event.ingested - external: ecs -- name: event.original - external: ecs -- name: event.created - external: ecs -- name: event.id - external: ecs -- name: event.kind - external: ecs -- name: event.severity - external: ecs -- name: event.url - external: ecs -- name: rule.id - external: ecs -- name: rule.name - external: ecs -- name: rule.ruleset - external: ecs -- name: rule.category - external: ecs -- name: user.name - external: ecs -- name: user.roles - external: ecs -- name: network.name - external: ecs -- name: error.message - external: ecs diff --git a/packages/zerofox/docs/README.md b/packages/zerofox/docs/README.md index 0c15bdf9bbc..c4aeddee448 100644 --- a/packages/zerofox/docs/README.md +++ b/packages/zerofox/docs/README.md @@ -15,63 +15,19 @@ Contains alert data received from the ZeroFox Cloud Platform | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | | dataset.name | Dataset name. | constant_keyword | | dataset.namespace | Dataset namespace. | constant_keyword | | dataset.type | Dataset type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | | event.dataset | Event dataset | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.url | URL linking to an external system to continue investigation of this event. This URL links to another system where in-depth investigation of the specific occurrence of this event can take place. Alert events, indicated by `event.kind:alert`, are a common use case for this field. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| network.name | Name given by operators to sections of their network. | keyword | -| rule.category | A categorization value keyword used by the entity using the rule for detection of this event. | keyword | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| rule.ruleset | Name of the ruleset, policy, group, or parent category in which the rule used to generate this event is a member. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user.roles | Array of user roles at the time of the event. | keyword | | zerofox.content_actions | | keyword | | zerofox.darkweb_term | | keyword | | zerofox.entity.entity_group.id | The entity group identifier. | integer | diff --git a/packages/zerofox/manifest.yml b/packages/zerofox/manifest.yml index d2e0226cb14..38a25dfe60e 100644 --- a/packages/zerofox/manifest.yml +++ b/packages/zerofox/manifest.yml @@ -1,6 +1,6 @@ name: zerofox title: ZeroFox -version: "1.24.0" +version: "1.25.0" description: Collect logs from ZeroFox with Elastic Agent. type: integration format_version: "3.0.2" @@ -13,7 +13,7 @@ categories: - security conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" policy_templates: - name: zerofox title: ZeroFox Alerts From d58c31ddcebc5ba9c9e72e2704f1d56be70a6e45 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:28:30 +0930 Subject: [PATCH 118/121] [zeronetworks] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/zeronetworks --- packages/zeronetworks/changelog.yml | 5 ++ .../data_stream/audit/fields/agent.yml | 50 ------------------- .../data_stream/audit/fields/ecs.yml | 24 --------- .../data_stream/audit/fields/fields.yml | 15 +++--- packages/zeronetworks/manifest.yml | 5 +- 5 files changed, 14 insertions(+), 85 deletions(-) delete mode 100644 packages/zeronetworks/data_stream/audit/fields/ecs.yml diff --git a/packages/zeronetworks/changelog.yml b/packages/zeronetworks/changelog.yml index c06b415bbd2..419ba71bd7c 100644 --- a/packages/zeronetworks/changelog.yml +++ b/packages/zeronetworks/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.15.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.14.0" changes: - description: Improve handling of empty responses. diff --git a/packages/zeronetworks/data_stream/audit/fields/agent.yml b/packages/zeronetworks/data_stream/audit/fields/agent.yml index b82e8558096..dbab79aaff6 100644 --- a/packages/zeronetworks/data_stream/audit/fields/agent.yml +++ b/packages/zeronetworks/data_stream/audit/fields/agent.yml @@ -1,59 +1,9 @@ -- name: cloud.account.id - external: ecs -- name: cloud.availability_zone - external: ecs -- name: cloud.instance.id - external: ecs -- name: cloud.instance.name - external: ecs -- name: cloud.machine.type - external: ecs -- name: cloud.project.id - external: ecs -- name: cloud.provider - external: ecs -- name: cloud.region - external: ecs -- name: container.id - external: ecs -- name: container.image.name - external: ecs -- name: container.labels - external: ecs -- name: container.name - external: ecs -- name: host.architecture - external: ecs - name: host.containerized type: boolean description: If the host is a container. -- name: host.domain - external: ecs -- name: host.hostname - external: ecs -- name: host.id - external: ecs -- name: host.ip - external: ecs -- name: host.mac - external: ecs -- name: host.name - external: ecs - name: host.os.build type: keyword description: OS build information. - name: host.os.codename type: keyword description: OS codename, if any. -- name: host.os.family - external: ecs -- name: host.os.kernel - external: ecs -- name: host.os.name - external: ecs -- name: host.os.platform - external: ecs -- name: host.os.version - external: ecs -- name: host.type - external: ecs diff --git a/packages/zeronetworks/data_stream/audit/fields/ecs.yml b/packages/zeronetworks/data_stream/audit/fields/ecs.yml deleted file mode 100644 index c245db21d10..00000000000 --- a/packages/zeronetworks/data_stream/audit/fields/ecs.yml +++ /dev/null @@ -1,24 +0,0 @@ -- name: ecs.version - external: ecs -- name: error.message - external: ecs -- name: event.code - external: ecs -- name: event.type - external: ecs -- name: event.category - external: ecs -- name: event.kind - external: ecs -- name: event.ingested - external: ecs -- name: event.original - external: ecs -- name: tags - external: ecs -- name: user.id - external: ecs -- name: user.full_name - external: ecs -- name: related.user - external: ecs diff --git a/packages/zeronetworks/data_stream/audit/fields/fields.yml b/packages/zeronetworks/data_stream/audit/fields/fields.yml index 1c3d1fda7ad..bea91ea35cc 100644 --- a/packages/zeronetworks/data_stream/audit/fields/fields.yml +++ b/packages/zeronetworks/data_stream/audit/fields/fields.yml @@ -92,7 +92,7 @@ type: integer description: > The current inactive reason. - + - name: clientId type: keyword description: > @@ -107,12 +107,12 @@ type: date description: > When the token expires. - + - name: externalIP type: ip description: > The external IP of the user. - + - name: idp type: integer description: > @@ -137,14 +137,13 @@ type: keyword description: > The name of the token - + - name: newAsset type: group description: > Fields for the asset of the audit. fields: - - name: id type: keyword description: > @@ -195,17 +194,17 @@ type: keyword description: > The type of token that was created. - + - name: uacId type: keyword description: > The UAC id. - + - name: uacName type: keyword description: > The UAC name. - + - name: user type: keyword description: > diff --git a/packages/zeronetworks/manifest.yml b/packages/zeronetworks/manifest.yml index 28e953765e0..e975ebe0f5f 100644 --- a/packages/zeronetworks/manifest.yml +++ b/packages/zeronetworks/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: zeronetworks title: "Zero Networks" -version: "1.14.0" +version: "1.15.0" source: license: "Elastic-2.0" description: "Zero Networks Logs integration" @@ -10,7 +10,7 @@ categories: - security conditions: kibana: - version: "^8.12.0" + version: "^8.13.0" elastic: subscription: "basic" screenshots: @@ -75,4 +75,3 @@ policy_templates: owner: github: elastic/security-service-integrations type: partner - From 3e6b2ac8a62cb3a8fd7e680a5c649225216012d1 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:28:33 +0930 Subject: [PATCH 119/121] [zoom] - Updated fields definitions The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/zoom --- packages/zoom/changelog.yml | 5 + .../zoom/data_stream/webhook/fields/agent.yml | 167 +----------------- .../zoom/data_stream/webhook/fields/ecs.yml | 78 -------- packages/zoom/docs/README.md | 72 -------- packages/zoom/manifest.yml | 4 +- 5 files changed, 8 insertions(+), 318 deletions(-) delete mode 100644 packages/zoom/data_stream/webhook/fields/ecs.yml diff --git a/packages/zoom/changelog.yml b/packages/zoom/changelog.yml index 6556912fd5a..53da81bec39 100644 --- a/packages/zoom/changelog.yml +++ b/packages/zoom/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.20.0" + changes: + - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.19.0" changes: - description: Set sensitive values as secret. diff --git a/packages/zoom/data_stream/webhook/fields/agent.yml b/packages/zoom/data_stream/webhook/fields/agent.yml index 845b84ed9c0..4bdd88d3cd7 100644 --- a/packages/zoom/data_stream/webhook/fields/agent.yml +++ b/packages/zoom/data_stream/webhook/fields/agent.yml @@ -5,180 +5,15 @@ footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' type: group fields: - - name: account.id - level: extended - type: keyword - ignore_above: 1024 - description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. - - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' - example: 666777888999 - - name: availability_zone - level: extended - type: keyword - ignore_above: 1024 - description: Availability zone in which this host is running. - example: us-east-1c - - name: instance.id - level: extended - type: keyword - ignore_above: 1024 - description: Instance ID of the host machine. - example: i-1234567890abcdef0 - - name: instance.name - level: extended - type: keyword - ignore_above: 1024 - description: Instance name of the host machine. - - name: machine.type - level: extended - type: keyword - ignore_above: 1024 - description: Machine type of the host machine. - example: t2.medium - - name: provider - level: extended - type: keyword - ignore_above: 1024 - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - example: aws - - name: region - level: extended - type: keyword - ignore_above: 1024 - description: Region in which this host is running. - example: us-east-1 - - name: project.id - type: keyword - description: Name of the project in Google Cloud. - name: image.id type: keyword description: Image ID for the cloud instance. -- name: container - title: Container - group: 2 - description: 'Container fields are used for meta information about the specific container that is the source of information. - - These fields help correlate data based containers from any runtime.' - type: group - fields: - - name: id - level: core - type: keyword - ignore_above: 1024 - description: Unique container id. - - name: image.name - level: extended - type: keyword - ignore_above: 1024 - description: Name of the image the container was built on. - - name: labels - level: extended - type: object - object_type: keyword - description: Image labels. - - name: name - level: extended - type: keyword - ignore_above: 1024 - description: Container name. - name: host title: Host group: 2 - description: 'A host is defined as a general computing instance. - - ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' type: group fields: - - name: architecture - level: core - type: keyword - ignore_above: 1024 - description: Operating system architecture. - example: x86_64 - - name: domain - level: extended - type: keyword - ignore_above: 1024 - description: 'Name of the domain of which the host is a member. - - For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' - example: CONTOSO - default_field: false - - name: hostname - level: core - type: keyword - ignore_above: 1024 - description: 'Hostname of the host. - - It normally contains what the `hostname` command returns on the host machine.' - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique host id. - - As hostname is not always unique, use values that are meaningful in your environment. - - Example: The current usage of `beat.name`.' - - name: ip - level: core - type: ip - description: Host ip addresses. - - name: mac - level: core - type: keyword - ignore_above: 1024 - description: Host mac addresses. - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Name of the host. - - It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' - - name: os.family - level: extended - type: keyword - ignore_above: 1024 - description: OS family (such as redhat, debian, freebsd, windows). - example: debian - - name: os.kernel - level: extended - type: keyword - ignore_above: 1024 - description: Operating system kernel version as a raw string. - example: 4.4.0-112-generic - - name: os.name - level: extended - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Operating system name, without the version. - example: Mac OS X - - name: os.platform - level: extended - type: keyword - ignore_above: 1024 - description: Operating system platform (such centos, ubuntu, windows). - example: darwin - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - example: 10.14.1 - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of host. - - For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' - name: containerized type: boolean description: > diff --git a/packages/zoom/data_stream/webhook/fields/ecs.yml b/packages/zoom/data_stream/webhook/fields/ecs.yml deleted file mode 100644 index b88c073dd9e..00000000000 --- a/packages/zoom/data_stream/webhook/fields/ecs.yml +++ /dev/null @@ -1,78 +0,0 @@ -- external: ecs - name: destination.user.id -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.id -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.outcome -- external: ecs - name: event.type -- external: ecs - name: message -- external: ecs - name: observer.product -- external: ecs - name: observer.vendor -- external: ecs - name: related.user -- external: ecs - name: source.user.id -- external: ecs - name: tags -- external: ecs - name: url.full -- external: ecs - name: user.changes.domain -- external: ecs - name: user.changes.email -- external: ecs - name: user.changes.full_name -- external: ecs - name: user.changes.group.domain -- external: ecs - name: user.changes.group.id -- external: ecs - name: user.changes.group.name -- external: ecs - name: user.changes.id -- external: ecs - name: user.changes.name -- external: ecs - name: user.domain -- external: ecs - name: user.email -- external: ecs - name: user.full_name -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user.target.domain -- external: ecs - name: user.target.email -- external: ecs - name: user.target.full_name -- external: ecs - name: user.target.group.domain -- external: ecs - name: user.target.group.id -- external: ecs - name: user.target.group.name -- external: ecs - name: user.target.id -- external: ecs - name: user.target.name diff --git a/packages/zoom/docs/README.md b/packages/zoom/docs/README.md index e19ef860ff7..73f2e418068 100644 --- a/packages/zoom/docs/README.md +++ b/packages/zoom/docs/README.md @@ -20,91 +20,19 @@ This integration is compatible with the Zoom Platform API as of September 2020. | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset name. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | | dataset.name | Dataset name. | constant_keyword | | dataset.namespace | Dataset namespace. | constant_keyword | | dataset.type | Dataset type. | constant_keyword | -| destination.user.id | Unique identifier of the user. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | | event.dataset | Event dataset | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type. | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| observer.product | The product name of the observer. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.user.id | Unique identifier of the user. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| url.full.text | Multi-field of `url.full`. | match_only_text | -| user.changes.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.changes.email | User email address. | keyword | -| user.changes.full_name | User's full name, if available. | keyword | -| user.changes.full_name.text | Multi-field of `user.changes.full_name`. | match_only_text | -| user.changes.group.domain | Name of the directory the group is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.changes.group.id | Unique identifier for the group on the system/platform. | keyword | -| user.changes.group.name | Name of the group. | keyword | -| user.changes.id | Unique identifier of the user. | keyword | -| user.changes.name | Short name or login of the user. | keyword | -| user.changes.name.text | Multi-field of `user.changes.name`. | match_only_text | -| user.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.email | User email address. | keyword | -| user.full_name | User's full name, if available. | keyword | -| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user.target.domain | Name of the directory the user is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.target.email | User email address. | keyword | -| user.target.full_name | User's full name, if available. | keyword | -| user.target.full_name.text | Multi-field of `user.target.full_name`. | match_only_text | -| user.target.group.domain | Name of the directory the group is a member of. For example, an LDAP or Active Directory domain name. | keyword | -| user.target.group.id | Unique identifier for the group on the system/platform. | keyword | -| user.target.group.name | Name of the group. | keyword | -| user.target.id | Unique identifier of the user. | keyword | -| user.target.name | Short name or login of the user. | keyword | -| user.target.name.text | Multi-field of `user.target.name`. | match_only_text | | zoom.account.account_alias | When an account alias is updated, this is the new value set | keyword | | zoom.account.account_name | When an account name is updated, this is the new value set | keyword | | zoom.account.account_support_email | When an account support_email is updated, this is the new value set | keyword | diff --git a/packages/zoom/manifest.yml b/packages/zoom/manifest.yml index 3517285db66..4907a006738 100644 --- a/packages/zoom/manifest.yml +++ b/packages/zoom/manifest.yml @@ -1,13 +1,13 @@ name: zoom title: Zoom -version: "1.19.0" +version: "1.20.0" description: Collect logs from Zoom with Elastic Agent. type: integration format_version: "3.0.2" categories: ["security", "productivity_security"] conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" policy_templates: - name: zoom title: Zoom logs From 3f1e14fb0993123bc57962e5f3790a91822ce405 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:28:37 +0930 Subject: [PATCH 120/121] [zscaler_zia] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.12.0 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/zscaler_zia --- packages/zscaler_zia/_dev/build/build.yml | 1 - packages/zscaler_zia/changelog.yml | 5 +++++ packages/zscaler_zia/data_stream/alerts/fields/beats.yml | 3 --- packages/zscaler_zia/data_stream/dns/fields/beats.yml | 3 --- packages/zscaler_zia/data_stream/firewall/fields/beats.yml | 3 --- packages/zscaler_zia/data_stream/tunnel/fields/beats.yml | 3 --- packages/zscaler_zia/data_stream/web/fields/beats.yml | 3 --- packages/zscaler_zia/docs/README.md | 5 ----- packages/zscaler_zia/manifest.yml | 4 ++-- 9 files changed, 7 insertions(+), 23 deletions(-) diff --git a/packages/zscaler_zia/_dev/build/build.yml b/packages/zscaler_zia/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/zscaler_zia/_dev/build/build.yml +++ b/packages/zscaler_zia/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/zscaler_zia/changelog.yml b/packages/zscaler_zia/changelog.yml index bacb447dfb9..0f0fa0e7329 100644 --- a/packages/zscaler_zia/changelog.yml +++ b/packages/zscaler_zia/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.20.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "2.19.2" changes: - description: Include cintip field to web log template. diff --git a/packages/zscaler_zia/data_stream/alerts/fields/beats.yml b/packages/zscaler_zia/data_stream/alerts/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/zscaler_zia/data_stream/alerts/fields/beats.yml +++ b/packages/zscaler_zia/data_stream/alerts/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/zscaler_zia/data_stream/dns/fields/beats.yml b/packages/zscaler_zia/data_stream/dns/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/zscaler_zia/data_stream/dns/fields/beats.yml +++ b/packages/zscaler_zia/data_stream/dns/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/zscaler_zia/data_stream/firewall/fields/beats.yml b/packages/zscaler_zia/data_stream/firewall/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/zscaler_zia/data_stream/firewall/fields/beats.yml +++ b/packages/zscaler_zia/data_stream/firewall/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/zscaler_zia/data_stream/tunnel/fields/beats.yml b/packages/zscaler_zia/data_stream/tunnel/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/zscaler_zia/data_stream/tunnel/fields/beats.yml +++ b/packages/zscaler_zia/data_stream/tunnel/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/zscaler_zia/data_stream/web/fields/beats.yml b/packages/zscaler_zia/data_stream/web/fields/beats.yml index 2d5ae254634..d5fd38748ba 100644 --- a/packages/zscaler_zia/data_stream/web/fields/beats.yml +++ b/packages/zscaler_zia/data_stream/web/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/zscaler_zia/docs/README.md b/packages/zscaler_zia/docs/README.md index 9e186317888..536d1a50718 100644 --- a/packages/zscaler_zia/docs/README.md +++ b/packages/zscaler_zia/docs/README.md @@ -250,7 +250,6 @@ An example event for `alerts` looks as following: | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | User defined tags. | keyword | | zscaler_zia.alerts.connection_lost_minutes | Amount of time after loosing connection to a server in Minutes. | double | | zscaler_zia.alerts.log_feed_name | Name of the NSS log feed. | keyword | @@ -383,7 +382,6 @@ An example event for `dns` looks as following: | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | User defined tags. | keyword | | zscaler_zia.dns.department | Department of the user. | keyword | | zscaler_zia.dns.dom.category | URL Category of the FQDN in the DNS request. | keyword | | zscaler_zia.dns.duration.milliseconds | Duration of the DNS request in milliseconds. | long | @@ -538,7 +536,6 @@ An example event for `firewall` looks as following: | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | User defined tags. | keyword | | zscaler_zia.firewall.aggregate | | keyword | | zscaler_zia.firewall.client.destination.ip | Client destination IP address. For aggregated sessions, this is the client destination IP address of the last session in the aggregate. | ip | | zscaler_zia.firewall.client.destination.port | Client destination port. For aggregated sessions, this is the client destination port of the last session in the aggregate. | long | @@ -666,7 +663,6 @@ An example event for `tunnel` looks as following: | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | User defined tags. | keyword | | zscaler_zia.tunnel.action.type | Type of the record. Possible values [ WL_TUNNEL_IPSECPHASE1, WL_TUNNEL_IPSECPHASE2, WL_TUNNEL_EVENT, WL_TUNNEL_SAMPLES ]. | keyword | | zscaler_zia.tunnel.authentication.algorithm | Authentication algorithm. | keyword | | zscaler_zia.tunnel.authentication.type | Authentication type. | keyword | @@ -868,7 +864,6 @@ An example event for `web` looks as following: | input.type | Type of Filebeat input. | keyword | | log.offset | Log offset. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | User defined tags. | keyword | | zscaler_zia.web.app.class | The web application class of the application that was accessed. Equivalent to module. | keyword | | zscaler_zia.web.app.name | Cloud application name. | keyword | | zscaler_zia.web.bandwidth_throttle | Indicates whether the transaction was throttled due to a configured bandwidth policy. | keyword | diff --git a/packages/zscaler_zia/manifest.yml b/packages/zscaler_zia/manifest.yml index 5b04bd492d5..12750b3c899 100644 --- a/packages/zscaler_zia/manifest.yml +++ b/packages/zscaler_zia/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: zscaler_zia title: Zscaler Internet Access -version: "2.19.2" +version: "2.20.0" description: Collect logs from Zscaler Internet Access (ZIA) with Elastic Agent. type: integration categories: @@ -11,7 +11,7 @@ source: license: "Elastic-2.0" conditions: kibana: - version: ^8.12.0 + version: "^8.13.0" elastic: subscription: "basic" screenshots: From fa913e11ff8a0768957b5eee8502ec8fe8d717b2 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 20 Jun 2024 13:28:41 +0930 Subject: [PATCH 121/121] [zscaler_zpa] - removed ecs import_mappings Removed import_mappings. The conditions.kibana.version in the package manifest changed from ^8.7.1 to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@v0.0.0-20240617213809-014b35dfe4c9 -ecs-version=8.11.0 -ecs-git-ref=git@v8.11.0 -drop-import-mappings -kibana-version=^8.13.0 -pr=10135 -fields-yml-drop-ecs packages/zscaler_zpa --- packages/zscaler_zpa/_dev/build/build.yml | 1 - packages/zscaler_zpa/changelog.yml | 5 +++++ .../data_stream/app_connector_status/fields/beats.yml | 3 --- packages/zscaler_zpa/data_stream/audit/fields/beats.yml | 3 --- .../zscaler_zpa/data_stream/browser_access/fields/beats.yml | 3 --- .../zscaler_zpa/data_stream/user_activity/fields/beats.yml | 3 --- .../zscaler_zpa/data_stream/user_status/fields/beats.yml | 3 --- packages/zscaler_zpa/docs/README.md | 5 ----- packages/zscaler_zpa/manifest.yml | 4 ++-- 9 files changed, 7 insertions(+), 23 deletions(-) diff --git a/packages/zscaler_zpa/_dev/build/build.yml b/packages/zscaler_zpa/_dev/build/build.yml index 71f48ba2a9c..2bfcfc223b0 100644 --- a/packages/zscaler_zpa/_dev/build/build.yml +++ b/packages/zscaler_zpa/_dev/build/build.yml @@ -1,4 +1,3 @@ dependencies: ecs: reference: "git@v8.11.0" - import_mappings: true diff --git a/packages/zscaler_zpa/changelog.yml b/packages/zscaler_zpa/changelog.yml index f982a44377d..800c14ccf37 100644 --- a/packages/zscaler_zpa/changelog.yml +++ b/packages/zscaler_zpa/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.18.0" + changes: + - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10135 - version: "1.17.0" changes: - description: Update manifest format version to v3.0.3. diff --git a/packages/zscaler_zpa/data_stream/app_connector_status/fields/beats.yml b/packages/zscaler_zpa/data_stream/app_connector_status/fields/beats.yml index 1214b97a0c7..28e147d9c90 100644 --- a/packages/zscaler_zpa/data_stream/app_connector_status/fields/beats.yml +++ b/packages/zscaler_zpa/data_stream/app_connector_status/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/zscaler_zpa/data_stream/audit/fields/beats.yml b/packages/zscaler_zpa/data_stream/audit/fields/beats.yml index 1214b97a0c7..28e147d9c90 100644 --- a/packages/zscaler_zpa/data_stream/audit/fields/beats.yml +++ b/packages/zscaler_zpa/data_stream/audit/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/zscaler_zpa/data_stream/browser_access/fields/beats.yml b/packages/zscaler_zpa/data_stream/browser_access/fields/beats.yml index 1214b97a0c7..28e147d9c90 100644 --- a/packages/zscaler_zpa/data_stream/browser_access/fields/beats.yml +++ b/packages/zscaler_zpa/data_stream/browser_access/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/zscaler_zpa/data_stream/user_activity/fields/beats.yml b/packages/zscaler_zpa/data_stream/user_activity/fields/beats.yml index 1214b97a0c7..28e147d9c90 100644 --- a/packages/zscaler_zpa/data_stream/user_activity/fields/beats.yml +++ b/packages/zscaler_zpa/data_stream/user_activity/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/zscaler_zpa/data_stream/user_status/fields/beats.yml b/packages/zscaler_zpa/data_stream/user_status/fields/beats.yml index 1214b97a0c7..28e147d9c90 100644 --- a/packages/zscaler_zpa/data_stream/user_status/fields/beats.yml +++ b/packages/zscaler_zpa/data_stream/user_status/fields/beats.yml @@ -4,6 +4,3 @@ - name: log.offset type: long description: Log offset. -- name: tags - type: keyword - description: User defined tags. diff --git a/packages/zscaler_zpa/docs/README.md b/packages/zscaler_zpa/docs/README.md index 23ebe780066..52befe3842a 100644 --- a/packages/zscaler_zpa/docs/README.md +++ b/packages/zscaler_zpa/docs/README.md @@ -128,7 +128,6 @@ Sample Response: | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | User defined tags. | keyword | | zscaler_zpa.app_connector_status.connector.group | The App Connector group name. | keyword | | zscaler_zpa.app_connector_status.connector.name | The App Connector name. | keyword | | zscaler_zpa.app_connector_status.connector_start_time | Time in seconds at which App Connector was started. | date | @@ -318,7 +317,6 @@ An example event for `app_connector_status` looks as following: | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | User defined tags. | keyword | | zscaler_zpa.audit.client_audit_update | The flag to represent if the event is a client Audit log. | long | | zscaler_zpa.audit.object.id | The ID associated with the object name. | keyword | | zscaler_zpa.audit.object.name | The name of the object. This corresponds to the Resource Name in the Audit Log page. | keyword | @@ -445,7 +443,6 @@ An example event for `audit` looks as following: | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | User defined tags. | keyword | | zscaler_zpa.browser_access.client_private_ip | The private IP address of the user's device. | ip | | zscaler_zpa.browser_access.connection.id | The application connection ID. | keyword | | zscaler_zpa.browser_access.connection.status | The status of the connection. | keyword | @@ -656,7 +653,6 @@ An example event for `browser_access` looks as following: | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | User defined tags. | keyword | | zscaler_zpa.user_activity.app_group | The application group name. | keyword | | zscaler_zpa.user_activity.app_learn_time | Time in microseconds taken for App Connectors to learn about the requested application and report the learned information to the central authority. | long | | zscaler_zpa.user_activity.application | The application name. | keyword | @@ -894,7 +890,6 @@ An example event for `user_activity` looks as following: | input.type | Type of filebeat input. | keyword | | log.offset | Log offset. | long | | log.source.address | Source address from which the log event was read / sent from. | keyword | -| tags | User defined tags. | keyword | | zscaler_zpa.user_status.client.type | The client type for the request (i.e., Zscaler Client Connector, ZPA LSS, or Web Browser). | keyword | | zscaler_zpa.user_status.fqdn.registered | The status of the hostname for the client-to-client connection. The expected values for this field are true or false. | boolean | | zscaler_zpa.user_status.fqdn.registered_error | The status of the registered hostname. | keyword | diff --git a/packages/zscaler_zpa/manifest.yml b/packages/zscaler_zpa/manifest.yml index e8334d439a1..31528c092c7 100644 --- a/packages/zscaler_zpa/manifest.yml +++ b/packages/zscaler_zpa/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: zscaler_zpa title: Zscaler Private Access -version: "1.17.0" +version: "1.18.0" source: license: Elastic-2.0 description: Collect logs from Zscaler Private Access (ZPA) with Elastic Agent. @@ -12,7 +12,7 @@ categories: - vpn_security conditions: kibana: - version: ^8.7.1 + version: "^8.13.0" elastic: subscription: basic screenshots: