[cisco_ise] Improve handling of unset fields from source (#10754)

Improve the handling of unset/null data fields, by checking for null before using the data in more processors. Some log messages do not set all expected fields. This adds more checks that data is not null attempting to run processors on these fields. This also adds some examples of log messages that do not populate some datafields to the pipeline tests. --------- Co-authored-by: Andrew Kroh <[email protected]>
elastic · Aug 14, 2024 · 18186e9 · 18186e9
1 parent 4523d07
commit 18186e9
Show file tree

Hide file tree

Showing 8 changed files with 211 additions and 7 deletions.
diff --git a/packages/cisco_ise/changelog.yml b/packages/cisco_ise/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.22.4"
+  changes:
+    - description: Improve handling of empty data fields
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10754
 - version: "1.22.3"
   changes:
     - description: Fix the Cisco_ISE toggle description for filestream input

diff --git a/...ges/cisco_ise/data_stream/log/_dev/test/pipeline/test-pipeline-passed-authentications.log b/...ges/cisco_ise/data_stream/log/_dev/test/pipeline/test-pipeline-passed-authentications.log
@@ -3,3 +3,4 @@
 <181>Feb 23 21:44:54 cisco-ise-host CISE_Passed_Authentications 0000000028 1 0 2021-02-23 21:44:54.276 +00:00 0000001707 5233 NOTICE Passed-Authentication: TrustSec Data Download Succeeded, ConfigVersionId=9, Device IP Address=81.2.69.144, DestinationIPAddress=81.2.69.144, DestinationPort=1645, UserName=#CTSREQUEST#, Protocol=Radius, RequestLatency=281, NetworkDeviceName=ASAv-vpn, User-Name=#CTSREQUEST#, NAS-IP-Address=81.2.69.144, NAS-Port=2, NAS-Port-Type=Virtual, cisco-av-pair=cts-environment-version=1, cisco-av-pair=cts-environment-data=ASAv-vpn, cisco-av-pair=cts-device-capability=env-data-fragment, cisco-av-pair=cts-pac-opaque=****, cisco-av-pair=coa-push=true, NetworkDeviceProfileName=Cisco, NetworkDeviceProfileId=8ade1f15-aef1-4a9a-8158-d02e835179db, IsThirdPartyDeviceFlow=false, AcsSessionID=ise/403491114/1, SelectedAccessService=NDAC_SGT_Service, Step=11001, Step=11017, Step=11117, Step=15012, Step=15036, Step=15006, Step=11002, NetworkDeviceGroups=Location#All Locations#dCloud, NetworkDeviceGroups=Device Type#All Device Types#Security Devices#VPN, AuthorizationPolicyMatchedRule=Default, CPMSessionID=c612851bJ4_5zUNfXSy7PCu6hSY3K1tPzLJOLXwVfJMIFdTrUjg, ISEPolicySetName=NetworkDeviceAuthorization, DTLSSupport=Unknown, Network Device Profile=Cisco, Location=Location#All Locations#dCloud, Device Type=Device Type#All Device Types#Security Devices#VPN, Response={Class=CACS:c612851bJ4_5zUNfXSy7PCu6hSY3K1tPzLJOLXwVfJMIFdTrUjg:ise/403491114/1; cisco-av-pair=cts:server-list=CTSServerList1-0001; cisco-av-pair=cts:security-group-tag=0002-11; cisco-av-pair=cts:environment-data-expiry=86400; cisco-av-pair=cts:security-group-table=0001-46; },
 <181>Mar 3 09:11:58 cisco-ise-host CISE_Passed_Authentications 0000082517 1 0 2022-03-03 09:11:58.729 +00:00 0000082584 5239 NOTICE RADIUS: NAS problem was fixed, ConfigVersionId=1626, NAS-IP-Address=81.2.69.145, MisconfiguredClientFixReason=Silent, Step=5239,
 <181>Mar 3 09:11:58 cisco-ise-host CISE_Passed_Authentications 0000082547 3 1 ConfigVersionId=1626, NAS-IP-Address=81.2.69.144, MisconfiguredClientFixReason=Silent, Step=5234,
+<181>Jul 1 06:49:05 cisco-ise-host CISE_Passed_Authentications 0006591647 18 11 Domain trust is one-way, StepData=120=fhgcg.local,Domain trust is one-way, StepData=121=fgfcx.local,Domain trust is one-way, StepData=122=gfhbnft.local,Domain trust is one-way, StepData=123=dthth.local,Domain trust is one-way, StepData=124=gkzjf.local,Domain trust is one-way, StepData=125=fjzhjhz.dfth-fzt.com,Domain trust is one-way, StepData=126=drzg.local,Domain trust is one-way, StepData=127=fzjh.local,Domain trust is one-way, StepData=128=zjn.local,Domain trust is one-way, StepData=129=fzjfg.jzg.de,Domain trust is one-way, StepData=130=gzjz.local,Domain trust is one-way, StepData=131=esfs.local,Domain trust is one-way, StepData=132=drghgh.local,Domain trust is one-way, StepData=133=rthhtd.local,Domain trust is one-way, StepData=134=rtzh.local,Domain trust is one-way, StepData=135=fzjfhj.local,Domain trust is one-way, StepData=136=kgzhf.local,Domain trust is one-way,
diff --git a/...data_stream/log/_dev/test/pipeline/test-pipeline-passed-authentications.log-expected.json b/...data_stream/log/_dev/test/pipeline/test-pipeline-passed-authentications.log-expected.json
@@ -640,6 +640,47 @@
             "tags": [
                 "preserve_original_event"
             ]
+        },
+        {
+            "@timestamp": "2024-07-01T06:49:05.000Z",
+            "cisco_ise": {
+                "log": {
+                    "category": {
+                        "name": "CISE_Passed_Authentications"
+                    },
+                    "message": {
+                        "id": "0006591647"
+                    },
+                    "segment": {
+                        "number": 11,
+                        "total": 18
+                    }
+                }
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "kind": "event",
+                "original": "<181>Jul 1 06:49:05 cisco-ise-host CISE_Passed_Authentications 0006591647 18 11 Domain trust is one-way, StepData=120=fhgcg.local,Domain trust is one-way, StepData=121=fgfcx.local,Domain trust is one-way, StepData=122=gfhbnft.local,Domain trust is one-way, StepData=123=dthth.local,Domain trust is one-way, StepData=124=gkzjf.local,Domain trust is one-way, StepData=125=fjzhjhz.dfth-fzt.com,Domain trust is one-way, StepData=126=drzg.local,Domain trust is one-way, StepData=127=fzjh.local,Domain trust is one-way, StepData=128=zjn.local,Domain trust is one-way, StepData=129=fzjfg.jzg.de,Domain trust is one-way, StepData=130=gzjz.local,Domain trust is one-way, StepData=131=esfs.local,Domain trust is one-way, StepData=132=drghgh.local,Domain trust is one-way, StepData=133=rthhtd.local,Domain trust is one-way, StepData=134=rtzh.local,Domain trust is one-way, StepData=135=fzjfhj.local,Domain trust is one-way, StepData=136=kgzhf.local,Domain trust is one-way,"
+            },
+            "host": {
+                "hostname": "cisco-ise-host"
+            },
+            "log": {
+                "syslog": {
+                    "priority": 181
+                }
+            },
+            "message": "Domain trust is one-way, StepData=120=fhgcg.local,Domain trust is one-way, StepData=121=fgfcx.local,Domain trust is one-way, StepData=122=gfhbnft.local,Domain trust is one-way, StepData=123=dthth.local,Domain trust is one-way, StepData=124=gkzjf.local,Domain trust is one-way, StepData=125=fjzhjhz.dfth-fzt.com,Domain trust is one-way, StepData=126=drzg.local,Domain trust is one-way, StepData=127=fzjh.local,Domain trust is one-way, StepData=128=zjn.local,Domain trust is one-way, StepData=129=fzjfg.jzg.de,Domain trust is one-way, StepData=130=gzjz.local,Domain trust is one-way, StepData=131=esfs.local,Domain trust is one-way, StepData=132=drghgh.local,Domain trust is one-way, StepData=133=rthhtd.local,Domain trust is one-way, StepData=134=rtzh.local,Domain trust is one-way, StepData=135=fzjfhj.local,Domain trust is one-way, StepData=136=kgzhf.local,Domain trust is one-way,",
+            "related": {
+                "hosts": [
+                    "cisco-ise-host"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         }
     ]
 }
diff --git a/packages/cisco_ise/data_stream/log/_dev/test/pipeline/test-pipeline-tacacs-accounting.log b/packages/cisco_ise/data_stream/log/_dev/test/pipeline/test-pipeline-tacacs-accounting.log
@@ -2,3 +2,4 @@
 <182>Feb 21 19:13:08 cisco-ise-host CISE_TACACS_Accounting 0000000001 4 0 2020-02-21 19:13:08.328 +00:00 0018415636 3301 NOTICE Tacacs-Accounting: TACACS+ Accounting START, ConfigVersionId=1829, Device IP Address=81.2.69.144, RequestLatency=1, NetworkDeviceName=LDNBuildSW1, Type=Accounting, Privilege-Level=15, Service=Login, User=psxlms, Port=tty2, Remote-Address=81.2.69.144, Authen-Method=TacacsPlus, AVPair=task_id=35585, AVPair=timezone=GMT, AVPair=start_time=1585222245, AcctRequest-Flags=Start, Service-Argument=shell, AcsSessionID=ldnnacpsn1/359344348/954422, SelectedAccessService=Device Admin - TACACS, Step=13006, Step=15049, Step=15008, Step=15048, Step=22083, Step=13035, NetworkDeviceGroups=Location#All Locations#EMEA, NetworkDeviceGroups=Device Type#All Device Types#Switches, NetworkDeviceGroups=IPSEC#Is IPSEC Device, CPMSessionID=81.2.69.144Accounting647817909, Model Name=Unknown, Software Version=Unknown, Network Device Profile=Cisco, Location=Location#All Locations#EMEA, Device Type=Device Type#All Device Types#Switches, IPSEC=IPSEC#Is IPSEC Device, Response={AcctReply-Status=Success; }, Device Type=Device Type#All Device Types#Switches, IPSEC=IPSEC#Is IPSEC Device, Response={AcctReply-Status=Success; }
 <182>Feb 21 19:13:08 cisco-ise-host CISE_TACACS_Accounting 0000000001 4 0 2020-02-21 19:13:08.328 +00:00 0018415932 3302 NOTICE Tacacs-Accounting: TACACS+ Accounting STOP, ConfigVersionId=1829, Device IP Address=81.2.69.144, RequestLatency=1, NetworkDeviceName=LDNBuildSW1, Type=Accounting, Privilege-Level=1, Service=Login, User=psxlms, Port=tty2, Remote-Address=81.2.69.144, Authen-Method=TacacsPlus, AVPair=task_id=35585, AVPair=timezone=GMT, AVPair=start_time=1585222245, AVPair=disc-cause=1, AVPair=disc-cause-ext=9, AVPair=pre-session-time=0, AVPair=elapsed_time=127, AVPair=stop_time=1585222372, AcctRequest-Flags=Stop, Service-Argument=shell, AcsSessionID=ldnnacpsn1/359344348/954446, SelectedAccessService=Device Admin - TACACS, Step=13006, Step=15049, Step=15008, Step=15048, Step=22084, Step=13035, NetworkDeviceGroups=Location#All Locations#EMEA, NetworkDeviceGroups=Device Type#All Device Types#Switches, NetworkDeviceGroups=IPSEC#Is IPSEC Device, CPMSessionID=81.2.69.144Accounting2791676098, Model Name=Unknown, Software Version=Unknown, Network Device Profile=Cisco, Location=Location#All Locations#EMEA, Device Type=Device Type#All Device Types#Switches, IPSEC=IPSEC#Is IPSEC Device, Response={AcctReply-Status=Success; }, Model Name=Unknown, Software Version=Unknown, Network Device Profile=Cisco, Location=Location#All Locations#EMEA, Device Type=Device Type#All Device Types#Switches, IPSEC=IPSEC#Is IPSEC Device, Response={AcctReply-Status=Success;}
 <182>Feb 21 19:13:08 cisco-ise-host CISE_TACACS_Accounting 0000000001 4 1 ConfigVersionId=1856, Device IP Address=81.2.69.144, RequestLatency=6, NetworkDeviceName=LDNBuildSW1, Type=Accounting, Privilege-Level=1, Service=Login, User=psxlms, Port=tty2, Remote-Address=81.2.69.144, Authen-Method=TacacsPlus, AVPair=task_id=35585, AVPair=timezone=GMT, AVPair=start_time=1585222245, AVPair=disc-cause=1, AVPair=disc-cause-ext=9, AVPair=pre-session-time=0, AVPair=elapsed_time=127, AVPair=stop_time=1585222372, AcctRequest-Flags=Stop, Service-Argument=shell, AcsSessionID=ldnnacpsn1/359344348/954446, SelectedAccessService=Device Admin - TACACS, Step=13006, Step=15049, Step=15008, Step=15048, Step=22084, Step=13035, NetworkDeviceGroups=Location#All Locations#EMEA, NetworkDeviceGroups=Device Type#All Device Types#Switches, NetworkDeviceGroups=IPSEC#Is IPSEC Device, Response={AcctReply-Status=Success; }, Model Name=Unknown, Software Version=Unknown, Network Device Profile=Cisco, Location=Location#All Locations#EMEA, Device Type=Device Type#All Device Types#Switches, IPSEC=IPSEC#Is IPSEC Device, Response={AcctReply-Status=Success;}
+<181>Jul 12 08:49:05 cisco-ise-host CISE_TACACS_Accounting 0006616665 2 0 2024-07-12 08:49:05.018 +02:00 17627964199 3300 NOTICE Tacacs-Accounting: TACACS+ Accounting with Command, ConfigVersionId=280, Device IP Address=81.2.69.144, CmdSet=[ CmdAV=show ip arp vrf vlan1111-vrf ], RequestLatency=5, NetworkDeviceName=rt333-rk000009, Type=Accounting, Privilege-Level=15, Service=Login, User=user, Port=tty3, Remote-Address=81.2.69.145, Authen-Method=TacacsPlus, AVPair=task_id=34866, AVPair=timezone=CEST, AVPair=start_time=1720766945, AVPair=priv-lvl=1, AcctRequest-Flags=Stop, Service-Argument=shell, AcsSessionID=mgtise001/498316448/86232573, SelectedAccessService=Default Device Admin, Step=13006, Step=15049, Step=15008, Step=15048, Step=15048, Step=13035, NetworkDeviceGroups=Location#All Locations#All41S#DC#EQ-FR7, NetworkDeviceGroups=Device Type#All Device Types#Router, NetworkDeviceGroups=IPSEC#Is IPSEC Device#No, CPMSessionID=111095075910.202.200.10013807Accounting1110950759, Network Device Profile=Cisco,
diff --git a/..._ise/data_stream/log/_dev/test/pipeline/test-pipeline-tacacs-accounting.log-expected.json b/..._ise/data_stream/log/_dev/test/pipeline/test-pipeline-tacacs-accounting.log-expected.json
@@ -619,6 +619,141 @@
             "user": {
                 "name": "psxlms"
             }
+        },
+        {
+            "@timestamp": "2024-07-12T08:49:05.018+02:00",
+            "cisco_ise": {
+                "log": {
+                    "acct": {
+                        "request": {
+                            "flags": "Stop"
+                        }
+                    },
+                    "acs": {
+                        "session": {
+                            "id": "mgtise001/498316448/86232573"
+                        }
+                    },
+                    "authen_method": "TacacsPlus",
+                    "avpair": {
+                        "priv_lvl": 1,
+                        "start_time": "2024-07-12T06:49:05.000Z",
+                        "task_id": "34866",
+                        "timezone": "CEST"
+                    },
+                    "category": {
+                        "name": "CISE_TACACS_Accounting"
+                    },
+                    "cmdset": "[ CmdAV=show ip arp vrf vlan1111-vrf ]",
+                    "config_version": {
+                        "id": 280
+                    },
+                    "cpm": {
+                        "session": {
+                            "id": "111095075910.202.200.10013807Accounting1110950759"
+                        }
+                    },
+                    "message": {
+                        "code": "3300",
+                        "description": "Tacacs-Accounting: TACACS+ Accounting with Command",
+                        "id": "0006616665"
+                    },
+                    "network": {
+                        "device": {
+                            "groups": [
+                                "Location#All Locations#All41S#DC#EQ-FR7",
+                                "Device Type#All Device Types#Router",
+                                "IPSEC#Is IPSEC Device#No"
+                            ],
+                            "name": "rt333-rk000009",
+                            "profile": "Cisco"
+                        }
+                    },
+                    "port": "tty3",
+                    "privilege": {
+                        "level": 15
+                    },
+                    "request": {
+                        "latency": 5
+                    },
+                    "segment": {
+                        "number": 0,
+                        "total": 2
+                    },
+                    "selected": {
+                        "access": {
+                            "service": "Default Device Admin"
+                        }
+                    },
+                    "service": {
+                        "argument": "shell",
+                        "name": "Login"
+                    },
+                    "step": [
+                        "13006",
+                        "15049",
+                        "15008",
+                        "15048",
+                        "15048",
+                        "13035"
+                    ],
+                    "type": "Accounting"
+                }
+            },
+            "client": {
+                "ip": "81.2.69.144"
+            },
+            "destination": {
+                "ip": "81.2.69.145"
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "action": "tacacs-accounting",
+                "category": [
+                    "configuration"
+                ],
+                "code": "3300",
+                "kind": "event",
+                "original": "<181>Jul 12 08:49:05 cisco-ise-host CISE_TACACS_Accounting 0006616665 2 0 2024-07-12 08:49:05.018 +02:00 17627964199 3300 NOTICE Tacacs-Accounting: TACACS+ Accounting with Command, ConfigVersionId=280, Device IP Address=81.2.69.144, CmdSet=[ CmdAV=show ip arp vrf vlan1111-vrf ], RequestLatency=5, NetworkDeviceName=rt333-rk000009, Type=Accounting, Privilege-Level=15, Service=Login, User=user, Port=tty3, Remote-Address=81.2.69.145, Authen-Method=TacacsPlus, AVPair=task_id=34866, AVPair=timezone=CEST, AVPair=start_time=1720766945, AVPair=priv-lvl=1, AcctRequest-Flags=Stop, Service-Argument=shell, AcsSessionID=mgtise001/498316448/86232573, SelectedAccessService=Default Device Admin, Step=13006, Step=15049, Step=15008, Step=15048, Step=15048, Step=13035, NetworkDeviceGroups=Location#All Locations#All41S#DC#EQ-FR7, NetworkDeviceGroups=Device Type#All Device Types#Router, NetworkDeviceGroups=IPSEC#Is IPSEC Device#No, CPMSessionID=111095075910.202.200.10013807Accounting1110950759, Network Device Profile=Cisco,",
+                "sequence": 17627964199,
+                "timezone": "+02:00",
+                "type": [
+                    "info"
+                ]
+            },
+            "host": {
+                "hostname": "cisco-ise-host"
+            },
+            "log": {
+                "level": "notice",
+                "syslog": {
+                    "priority": 181,
+                    "severity": {
+                        "name": "notice"
+                    }
+                }
+            },
+            "message": "2024-07-12 08:49:05.018 +02:00 17627964199 3300 NOTICE Tacacs-Accounting: TACACS+ Accounting with Command, ConfigVersionId=280, Device IP Address=81.2.69.144, CmdSet=[ CmdAV=show ip arp vrf vlan1111-vrf ], RequestLatency=5, NetworkDeviceName=rt333-rk000009, Type=Accounting, Privilege-Level=15, Service=Login, User=user, Port=tty3, Remote-Address=81.2.69.145, Authen-Method=TacacsPlus, AVPair=task_id=34866, AVPair=timezone=CEST, AVPair=start_time=1720766945, AVPair=priv-lvl=1, AcctRequest-Flags=Stop, Service-Argument=shell, AcsSessionID=mgtise001/498316448/86232573, SelectedAccessService=Default Device Admin, Step=13006, Step=15049, Step=15008, Step=15048, Step=15048, Step=13035, NetworkDeviceGroups=Location#All Locations#All41S#DC#EQ-FR7, NetworkDeviceGroups=Device Type#All Device Types#Router, NetworkDeviceGroups=IPSEC#Is IPSEC Device#No, CPMSessionID=111095075910.202.200.10013807Accounting1110950759, Network Device Profile=Cisco,",
+            "related": {
+                "hosts": [
+                    "cisco-ise-host"
+                ],
+                "ip": [
+                    "81.2.69.144",
+                    "81.2.69.145"
+                ],
+                "user": [
+                    "user"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user": {
+                "name": "user"
+            }
         }
     ]
 }