Skip to content

Commit

Permalink
azure_frontdoor: Add identity field to access and waf datastreams. (#…
Browse files Browse the repository at this point in the history 
…10689)

Add `identity` field to access and waf datastreams.

The field can be string or object. If it is string, the field is renamed 
to `identity_name`. Otherwise it is stored as an object.

   - User fields are derived from `identity` field.

   - URL fields are derived using `uri_parts`.

   - Dynamic fields config on `url.extension` is added to pipeline test
     config to prevent flaky test issue when running with stack > 8.14+.
  • Loading branch information
@kcreddy
kcreddy authored Aug 6, 2024
1 parent bc2c24a commit 0e4459d
Show file tree
Hide file tree
Showing 13 changed files with 1,263 additions and 8 deletions.
5 changes: 5 additions & 0 deletions packages/azure_frontdoor/changelog.yml
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
- version: "1.9.0"
changes:
- description: Add new field identity.
type: enhancement
link: https://github.com/elastic/integrations/pull/10689
- version: "1.8.0"
changes:
- description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.
Expand Down
3 changes: 3 additions & 0 deletions packages/azure_frontdoor/data_stream/access/_dev/test/pipeline/test-common-config.yml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
dynamic_fields:
"event.ingested": ".*"
# This can be removed after ES 8.14 is the minimum version.
# Relates: https://github.com/elastic/elasticsearch/pull/105689
url.extension: '^.*$'
fields:
tags:
- preserve_original_event
Expand Down
5 changes: 4 additions & 1 deletion packages/azure_frontdoor/data_stream/access/_dev/test/pipeline/test-fdaccess.log
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
{"category":"FrontdoorAccessLog","operationName":"Microsoft.Network/FrontDoor/AccessLog/Write","properties":{"ErrorInfo":"NoError","backendHostname":"samplev6erp.azurewebsites.net:443","cacheStatus":"CONFIG_NOCACHE","clientIp":"89.160.20.128","clientPort":"50382","httpMethod":"POST","httpStatusCode":"200","httpStatusDetails":"200","httpVersion":"2.0.0.0","isReceivedFromClient":true,"pop":"SIN","requestBytes":"2545","requestProtocol":"HTTPS","requestUri":"https://erp.testcloud.com:443/StockSetup/GetStockListByCir","responseBytes":"1205","routingRuleName":"erp","rulesEngineMatchNames":[],"securityProtocol":"TLS 1.2","socketIp":"89.160.20.128","timeTaken":"0.384","timeToFirstByte":"0.384","trackingReference":"0k1y5YQAAAAAWd0Uc6UcnR7WN8uo2prYZU0lOMzBFREdFMDIxNwBkYjIxMTMyNi1mZTJmLTQwYWYtOTA4My03MGUyYTJmYWRmZjc=","userAgent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36"},"resourceId":"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.NETWORK/FRONTDOORS/TESTCLOUD","time":"2021-12-15T03:10:11.6479719Z"}
{"category":"FrontdoorAccessLog","operationName":"Microsoft.Network/FrontDoor/AccessLog/Write","properties":{"ErrorInfo":"NoError","backendHostname":"samplev6erp.azurewebsites.net:443","cacheStatus":"CONFIG_NOCACHE","clientIp":"175.16.199.0","clientPort":"6610","httpMethod":"GET","httpStatusCode":"200","httpStatusDetails":"200","httpVersion":"2.0.0.0","isReceivedFromClient":true,"pop":"SIN","requestBytes":"1984","requestProtocol":"HTTPS","requestUri":"https://erp.testcloud.com:443/saleInvoice/readBySyskeySIByRoleAllowed/2112140619239361392","responseBytes":"2308","routingRuleName":"erp","rulesEngineMatchNames":[],"securityProtocol":"TLS 1.2","socketIp":"175.16.199.0","timeTaken":"0.122","timeToFirstByte":"0.122","trackingReference":"0lWK5YQAAAAD89Q/jewlnT7dWvZNIh72LU0lOMzBFREdFMDIxNwBkYjIxMTMyNi1mZTJmLTQwYWYtOTA4My03MGUyYTJmYWRmZjc=","userAgent":"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36"},"resourceId":"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.NETWORK/FRONTDOORS/TESTCLOUD","time":"2021-12-15T03:35:49.9266300Z"}
{"category":"FrontdoorAccessLog","operationName":"Microsoft.Network/FrontDoor/AccessLog/Write","properties":{"ErrorInfo":"NoError","backendHostname":"samplev6erp.azurewebsites.net:443","cacheStatus":"CONFIG_NOCACHE","clientIp":"175.16.199.0","clientPort":"6610","httpMethod":"GET","httpStatusCode":"200","httpStatusDetails":"200","httpVersion":"2.0.0.0","isReceivedFromClient":true,"pop":"SIN","requestBytes":"1971","requestProtocol":"HTTPS","requestUri":"https://erp.testcloud.com:443/Customer/searchContactList/2107050813256062892","responseBytes":"637","routingRuleName":"erp","rulesEngineMatchNames":[],"securityProtocol":"TLS 1.2","socketIp":"175.16.199.0","timeTaken":"0.064","timeToFirstByte":"0.064","trackingReference":"0lWK5YQAAAAAnKnstK4rwSovl+unjuKhoU0lOMzBFREdFMDIxNwBkYjIxMTMyNi1mZTJmLTQwYWYtOTA4My03MGUyYTJmYWRmZjc=","userAgent":"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36"},"resourceId":"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.NETWORK/FRONTDOORS/TESTCLOUD","time":"2021-12-15T03:35:50.0584922Z"}
{''"records"'': [{"time":"2021-02-02T07:15:37.3640748Z","resourceId":"/SUBSCRIPTIONS/saDFEEQW-JESSIE","category":"FrontdoorAccessLog"}]}
{"category":"FrontdoorAccessLog","operationName":"Microsoft.Network/FrontDoor/AccessLog/Write","properties":{"ErrorInfo":"NoError","backendHostname":"samplev6erp.azurewebsites.net:443","cacheStatus":"CONFIG_NOCACHE","clientIp":"175.16.199.1","clientPort":"6611","httpMethod":"GET","httpStatusCode":"200","httpStatusDetails":"200","httpVersion":"2.0.0.0","isReceivedFromClient":true,"pop":"SIN","requestBytes":"1971","requestProtocol":"HTTPS","requestUri":"https://erp.testcloud.com:443/Customer/searchContactList/2107050813256062892","responseBytes":"637","routingRuleName":"erp","rulesEngineMatchNames":[],"securityProtocol":"TLS 1.2","socketIp":"175.16.199.0","timeTaken":"0.064","timeToFirstByte":"0.064","trackingReference":"0lWK5YQAAAAAnKnstK4rwSovl+unjuKhoU0lOMzBFREdFMDIxNwBkYjIxMTMyNi1mZTJmLTQwYWYtOTA4My03MGUyYTJmYWRmZjc=","userAgent":"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36"},"resourceId":"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.NETWORK/FRONTDOORS/TESTCLOUD","time":"2024-07-15T03:35:50.0584922Z","identity":"bobert"}
{"category":"FrontdoorAccessLog","operationName":"Microsoft.Network/FrontDoor/AccessLog/Write","properties":{"ErrorInfo":"NoError","backendHostname":"samplev6erp.azurewebsites.net:443","cacheStatus":"CONFIG_NOCACHE","clientIp":"175.16.199.2","clientPort":"6612","httpMethod":"GET","httpStatusCode":"200","httpStatusDetails":"200","httpVersion":"2.0.0.0","isReceivedFromClient":true,"pop":"SIN","requestBytes":"1971","requestProtocol":"HTTPS","requestUri":"https://erp.testcloud.com:443/Customer/searchContactList/2107050813256062892","responseBytes":"637","routingRuleName":"erp","rulesEngineMatchNames":[],"securityProtocol":"TLS 1.2","socketIp":"175.16.199.0","timeTaken":"0.064","timeToFirstByte":"0.064","trackingReference":"0lWK5YQAAAAAnKnstK4rwSovl+unjuKhoU0lOMzBFREdFMDIxNwBkYjIxMTMyNi1mZTJmLTQwYWYtOTA4My03MGUyYTJmYWRmZjc=","userAgent":"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36"},"resourceId":"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.NETWORK/FRONTDOORS/TESTCLOUD","time":"2024-07-20T03:35:50.0584922Z","identity":{"authorization":{"scope":"/subscriptions/s1/resourceGroups/MSSupportGroup/providers/microsoft.support/supporttickets/115012112305841","action":"microsoft.support/supporttickets/write","evidence":{"role":"Subscription Admin"}},"claims":{"aud":"https://management.core.windows.net/","iss":"https://sts.windows.net/72f988bf-86f1-41af-91ab-2d7cd011db47/","iat":"1421876371","nbf":"1421876371","exp":"1421880271","ver":"1.0","http://schemas.microsoft.com/identity/claims/tenantid":"00000000-0000-0000-0000-000000000000","http://schemas.microsoft.com/claims/authnmethodsreferences":"pwd","http://schemas.microsoft.com/identity/claims/objectidentifier":"2468adf0-8211-44e3-95xq-85137af64708","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn":"[email protected]","puid":"20030000801A118C","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier":"9vckmEGF7zDKk1YzIY8k0t1_EAPaXoeHyPRn6f413zM","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname":"John","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname":"Smith","name":"John Smith","groups":"cacfe77c-e058-4712-83qw-f9b08849fd60,7f71d11d-4c41-4b23-99d2-d32ce7aa621c,31522864-0578-4ea0-9gdc-e66cc564d18c","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name":" [email protected]","appid":"c44b4083-3bq0-49c1-b47d-974e53cbdf3c","appidacr":"2","http://schemas.microsoft.com/identity/claims/scope":"user_impersonation","http://schemas.microsoft.com/claims/authnclassreference":"1"}}}
{"category":"FrontdoorAccessLog","operationName":"Microsoft.Network/FrontDoor/AccessLog/Write","properties":{"ErrorInfo":"NoError","backendHostname":"samplev6erp.azurewebsites.net:443","cacheStatus":"CONFIG_NOCACHE","clientIp":"175.16.199.3","clientPort":"6613","httpMethod":"GET","httpStatusCode":"200","httpStatusDetails":"200","httpVersion":"2.0.0.0","isReceivedFromClient":true,"pop":"SIN","requestBytes":"1971","requestProtocol":"HTTPS","requestUri":"https://erp.testcloud.com:443/Customer/searchContactList/2107050813256062892","responseBytes":"637","routingRuleName":"erp","rulesEngineMatchNames":[],"securityProtocol":"TLS 1.2","socketIp":"175.16.199.0","timeTaken":"0.064","timeToFirstByte":"0.064","trackingReference":"0lWK5YQAAAAAnKnstK4rwSovl+unjuKhoU0lOMzBFREdFMDIxNwBkYjIxMTMyNi1mZTJmLTQwYWYtOTA4My03MGUyYTJmYWRmZjc=","userAgent":"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36"},"resourceId":"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.NETWORK/FRONTDOORS/TESTCLOUD","time":"2024-07-20T03:35:50.0584922Z","identity":{"authorization":{"scope":"/subscriptions/s1/resourceGroups/MSSupportGroup/providers/microsoft.support/supporttickets/115012112305841","action":"microsoft.support/supporttickets/write","evidence":{"principalId":"redacted","principalType":"ServicePrincipal","role":"Contributor","roleAssignmentId":"redacted","roleAssignmentScope":"/subscriptions/redacted","roleDefinitionId":"redacted"}},"claims":{"aud":"https://management.core.windows.net/","iss":"https://sts.windows.net/72f988bf-86f1-41af-91ab-2d7cd011db47/","iat":"1421876371","nbf":"1421876371","exp":"1421880271","ver":"1.0","http://schemas.microsoft.com/identity/claims/tenantid":"00000000-0000-0000-0000-000000000000","http://schemas.microsoft.com/claims/authnmethodsreferences":"pwd","http://schemas.microsoft.com/identity/claims/objectidentifier":"2468adf0-8211-44e3-95xq-85137af64708","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn":"[email protected]","puid":"20030000801A118C","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier":"9vckmEGF7zDKk1YzIY8k0t1_EAPaXoeHyPRn6f413zM","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname":"John","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname":"Smith","name":"John Smith","groups":"cacfe77c-e058-4712-83qw-f9b08849fd60,7f71d11d-4c41-4b23-99d2-d32ce7aa621c,31522864-0578-4ea0-9gdc-e66cc564d18c","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name":" [email protected]","appid":"c44b4083-3bq0-49c1-b47d-974e53cbdf3c","appidacr":"2","http://schemas.microsoft.com/identity/claims/scope":"user_impersonation","http://schemas.microsoft.com/claims/authnclassreference":"1"}}}
{''"records"'': [{"time":"2021-02-02T07:15:37.3640748Z","resourceId":"/SUBSCRIPTIONS/saDFEEQW-JESSIE","category":"FrontdoorAccessLog"}]}
Loading

0 comments on commit 0e4459d

Please sign in to comment.