From 6cf4b2af107a6f5d9e15a349a06ec91f75080609 Mon Sep 17 00:00:00 2001 From: Greg Shepherd Date: Sun, 29 Sep 2024 21:27:55 -0400 Subject: [PATCH] ti_threatconnect: add in technique.name field to the transform and remove milliseconds from TQL query. (#11217) * Add in technique.name field to the transform that came from a customer request. * Remove milliseconds from TQL query as current code drops the last 0 in the milliseconds which causes as issue. --- packages/ti_threatconnect/changelog.yml | 8 ++--- .../pipeline/test-indicator.log-expected.json | 5 +++- .../indicator/agent/stream/cel.yml.hbs | 4 ++- .../elasticsearch/ingest_pipeline/default.yml | 29 +++++++++++++++++-- .../transform/latest/fields/ecs.yml | 4 ++- packages/ti_threatconnect/manifest.yml | 2 +- 6 files changed, 42 insertions(+), 10 deletions(-) diff --git a/packages/ti_threatconnect/changelog.yml b/packages/ti_threatconnect/changelog.yml index 0e01a15181a..6ece2cca1a1 100644 --- a/packages/ti_threatconnect/changelog.yml +++ b/packages/ti_threatconnect/changelog.yml @@ -1,9 +1,9 @@ # newer versions go on top -- version: "1.2.3" +- version: "1.3.0" changes: - - description: Fix labels.is_ioc_transform_source values - type: bugfix - link: https://github.com/elastic/integrations/pull/11231 + - description: Add in technique.name field to the transform. Remove milliseconds from TQL query. + type: enhancement + link: https://github.com/elastic/integrations/pull/11217 - version: "1.2.2" changes: - description: Add missing fields in transform diff --git a/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-indicator.log-expected.json b/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-indicator.log-expected.json index b3992ed2234..a3175bb73d6 100644 --- a/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-indicator.log-expected.json +++ b/packages/ti_threatconnect/data_stream/indicator/_dev/test/pipeline/test-indicator.log-expected.json @@ -92,6 +92,9 @@ "technique": { "id": [ "T1055.005" + ], + "name": [ + "userexecution:maliciouslink" ] } }, @@ -537,4 +540,4 @@ } } ] -} \ No newline at end of file +} diff --git a/packages/ti_threatconnect/data_stream/indicator/agent/stream/cel.yml.hbs b/packages/ti_threatconnect/data_stream/indicator/agent/stream/cel.yml.hbs index 08c6c7da0a9..9f507b6190c 100644 --- a/packages/ti_threatconnect/data_stream/indicator/agent/stream/cel.yml.hbs +++ b/packages/ti_threatconnect/data_stream/indicator/agent/stream/cel.yml.hbs @@ -64,10 +64,12 @@ redact: - secret_key # The program section is where the logic of the stream processor is defined. +# Notice the format for the last timestamp does not include milliseconds. The default format included +# milliseconds and if it ended in 0, that 0 would be dropped and TC TQL then would error on the timestamp. program: | ['lastModified GEQ "'+( !state.want_more ? - state.?cursor.last_timestamp.orValue(string(now - duration(state.initial_interval))) + state.?cursor.last_timestamp.orValue((now - duration(state.initial_interval)).format(time_layout.RFC3339)) : state.?cursor.first_timestamp.orValue("") )+'"'+( diff --git a/packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml b/packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml index 0fce2ea1af3..8cb6aa46f78 100644 --- a/packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml @@ -1985,7 +1985,31 @@ processors: - remove: field: _ingest._value.lastUsed ignore_missing: true -# Some tags are Attack Patterns, rename the techniqueId to technique.id + # Some tags are Attack Patterns, map technique name to ECS field if technique ID exists + - script: + lang: painless + description: "Extract Technique Name from tags" + if: ctx.json?.tags?.data != null + tag: script_technique_names + source: |- + def t_names = new ArrayList(); + for (def obj : ctx.json.tags.data) { + if (obj.get('techniqueId') != null) { + t_names.add(obj.get('name')); + } + } + if (t_names.size() > 0){ + if (ctx.threat.technique == null) { + ctx.threat.technique = new HashMap(); + } + ctx.threat.technique.name = t_names; + } + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + +# Some tags are Attack Patterns, rename the technique id field for ES standardization - foreach: field: json.tags.data if: ctx.json?.tags?.data != null @@ -1996,7 +2020,7 @@ processors: tag: rename_tags_data_techniqueId target_field: _ingest._value.technique.id ignore_missing: true -# Append the current attack pattern technique id to the threat.technique.id field to capture the full list +# Some tags are Attack Patterns, map technique id to ECS field - foreach: field: json.tags.data if: ctx.json?.tags?.data != null @@ -2007,6 +2031,7 @@ processors: tag: append_tags_data_techniqueId value: '{{{_ingest._value.technique.id}}}' allow_duplicates: false + # For Attack pattern tags, capture the platform data and append to os.family - foreach: field: json.tags.data diff --git a/packages/ti_threatconnect/elasticsearch/transform/latest/fields/ecs.yml b/packages/ti_threatconnect/elasticsearch/transform/latest/fields/ecs.yml index 0726e9575e4..597c457bd6e 100644 --- a/packages/ti_threatconnect/elasticsearch/transform/latest/fields/ecs.yml +++ b/packages/ti_threatconnect/elasticsearch/transform/latest/fields/ecs.yml @@ -60,9 +60,11 @@ type: keyword - name: threat.technique.id type: keyword +- name: threat.technique.name + type: keyword - name: user_agent.original type: keyword -# Below fields to be moved into base-fields.yml after kibana.version changed to >= 8.14 +# Below fields to be moved into base-fields.yml after kibana.version changed to >= 8.14 # Related to fix: https://github.com/elastic/kibana/pull/177608 - name: event.module type: constant_keyword diff --git a/packages/ti_threatconnect/manifest.yml b/packages/ti_threatconnect/manifest.yml index 41febd9e429..fd99819f5fe 100644 --- a/packages/ti_threatconnect/manifest.yml +++ b/packages/ti_threatconnect/manifest.yml @@ -2,7 +2,7 @@ format_version: 3.0.3 name: ti_threatconnect title: ThreatConnect -version: "1.2.3" +version: "1.3.0" description: Collects Indicators from ThreatConnect using the Elastic Agent and saves them as logs inside Elastic type: integration categories: