-
Notifications
You must be signed in to change notification settings - Fork 458
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[MongoDB Atlas] Disk data stream (#10555)
* add disk data stream * update readme * update readme * address review coments * address review comments * address review comments * address review comments
- Loading branch information
1 parent
881c579
commit e642c8e
Showing
25 changed files
with
2,210 additions
and
56 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file modified
BIN
+433 KB
(110%)
packages/mongodb_atlas/_dev/deploy/docker/mongodb_atlas/test
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 2 additions & 0 deletions
2
packages/mongodb_atlas/data_stream/disk/_dev/test/pipeline/test-common-config.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| dynamic_fields: | ||
| "event.ingested": ".*" |
30 changes: 30 additions & 0 deletions
30
packages/mongodb_atlas/data_stream/disk/_dev/test/pipeline/test-disk-metrics.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| { | ||
| "events": [ | ||
| { | ||
| "hostId": "host-1", | ||
| "groupId": "group-1", | ||
| "response": { | ||
| "MAX_DISK_PARTITION_IOPS_TOTAL": 0.2138048, | ||
| "DISK_PARTITION_IOPS_READ": 0.33244343545, | ||
| "MAX_DISK_PARTITION_IOPS_READ": 0.3422309, | ||
| "DISK_PARTITION_IOPS_TOTAL": 7.38912, | ||
| "DISK_PARTITION_IOPS_WRITE": 5.2244123, | ||
| "MAX_DISK_PARTITION_IOPS_WRITE": 44.125672, | ||
| "MAX_DISK_PARTITION_LATENCY_READ": 1.321539, | ||
| "DISK_PARTITION_LATENCY_READ": 1.500023, | ||
| "MAX_DISK_PARTITION_LATENCY_WRITE": 1.2394742, | ||
| "DISK_PARTITION_LATENCY_WRITE": 0.5678899, | ||
| "DISK_PARTITION_SPACE_FREE": 6.33455633344, | ||
| "MAX_DISK_PARTITION_SPACE_FREE": 6.182233244, | ||
| "MAX_DISK_PARTITION_SPACE_PERCENT_FREE": 73.2324, | ||
| "DISK_PARTITION_SPACE_PERCENT_FREE": 71.23445789, | ||
| "DISK_PARTITION_SPACE_USED": 1.27898776, | ||
| "MAX_DISK_PARTITION_SPACE_USED": 25.127876554, | ||
| "MAX_DISK_PARTITION_SPACE_PERCENT_USED": 25.197166, | ||
| "DISK_PARTITION_SPACE_PERCENT_USED": 25.23388449 | ||
| }, | ||
| "partitionName": "data", | ||
| "processId": "hostname-1" | ||
| } | ||
| ] | ||
| } |
83 changes: 83 additions & 0 deletions
83
...es/mongodb_atlas/data_stream/disk/_dev/test/pipeline/test-disk-metrics.json-expected.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,83 @@ | ||
| { | ||
| "expected": [ | ||
| { | ||
| "ecs": { | ||
| "version": "8.11.0" | ||
| }, | ||
| "event": { | ||
| "category": [ | ||
| "database" | ||
| ], | ||
| "kind": "metric", | ||
| "module": "mongodb_atlas", | ||
| "type": [ | ||
| "info" | ||
| ] | ||
| }, | ||
| "group": { | ||
| "id": "group-1" | ||
| }, | ||
| "mongodb_atlas": { | ||
| "disk": { | ||
| "read": { | ||
| "iops": { | ||
| "max": { | ||
| "throughput": 0.3422309 | ||
| }, | ||
| "throughput": 0.33244343545 | ||
| }, | ||
| "latency": { | ||
| "max": { | ||
| "ms": 1.321539 | ||
| }, | ||
| "ms": 1.500023 | ||
| } | ||
| }, | ||
| "space": { | ||
| "free": { | ||
| "bytes": 6.33455633344, | ||
| "max": { | ||
| "bytes": 6.182233244, | ||
| "pct": 73.2324 | ||
| }, | ||
| "pct": 71.23445789 | ||
| }, | ||
| "used": { | ||
| "bytes": 1.27898776, | ||
| "max": { | ||
| "bytes": 25.127876554, | ||
| "pct": 25.197166 | ||
| }, | ||
| "pct": 25.23388449 | ||
| } | ||
| }, | ||
| "total": { | ||
| "iops": { | ||
| "max": { | ||
| "throughput": 0.2138048 | ||
| }, | ||
| "throughput": 7.38912 | ||
| } | ||
| }, | ||
| "write": { | ||
| "iops": { | ||
| "max": { | ||
| "throughput": 44.125672 | ||
| }, | ||
| "throughput": 5.2244123 | ||
| }, | ||
| "latency": { | ||
| "max": { | ||
| "ms": 1.2394742 | ||
| }, | ||
| "ms": 0.5678899 | ||
| } | ||
| } | ||
| }, | ||
| "host_id": "host-1", | ||
| "partition_name": "data", | ||
| "process_id": "hostname-1" | ||
| } | ||
| } | ||
| ] | ||
| } |
9 changes: 9 additions & 0 deletions
9
packages/mongodb_atlas/data_stream/disk/_dev/test/system/test-default-config.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| vars: | ||
| url: http://{{Hostname}}:{{Port}} | ||
| public_key: admin | ||
| private_key: MongoDB@123 | ||
| data_stream: | ||
| vars: | ||
| groupId: mongodb-group1 | ||
| input: cel | ||
| service: mongodbatlas |
166 changes: 166 additions & 0 deletions
166
packages/mongodb_atlas/data_stream/disk/agent/stream/input.yml.hbs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,166 @@ | ||
| config_version: 2 | ||
| interval: {{period}} | ||
| {{#if enable_request_tracer}} | ||
| resource.tracer.filename: "../../logs/cel/http-request-trace-*.ndjson" | ||
| {{/if}} | ||
| {{#if ssl}} | ||
| resource.ssl: {{ssl}} | ||
| {{/if}} | ||
| {{#if http_client_timeout}} | ||
| resource.timeout: {{http_client_timeout}} | ||
| {{/if}} | ||
| tags: | ||
| {{#if preserve_original_event}} | ||
| - preserve_original_event | ||
| {{/if}} | ||
| {{#each tags as |tag|}} | ||
| - {{tag}} | ||
| {{/each}} | ||
| {{#contains "forwarded" tags}} | ||
| publisher_pipeline.disable_host: true | ||
| {{/contains}} | ||
| {{#if processors}} | ||
| processors: | ||
| {{processors}} | ||
| {{/if}} | ||
| auth.digest: | ||
| user: {{public_key}} | ||
| password: {{private_key}} | ||
| resource.url: {{url}} | ||
| state: | ||
| group_id: {{groupId}} | ||
| want_more: false | ||
| page_num: 1 | ||
| disk_page_num: 1 | ||
| query: /measurements?granularity=PT{{period}}&period=PT{{period}} | ||
| redact: | ||
| fields: ~ | ||
| program: | | ||
| ( | ||
| (has(state.host_list) && size(state.host_list) > 0) ? | ||
| state | ||
| : | ||
| state.with( | ||
| request( | ||
| "GET", | ||
| state.url.trim_right("/") + "/api/atlas/v2/groups/" + state.group_id + "/processes?" + { | ||
| "pageNum": [string(state.page_num)], | ||
| "itemsPerPage": ["100"], | ||
| }.format_query() | ||
| ).with({ | ||
| "Header": { | ||
| "Accept": ["application/vnd.atlas." + string(timestamp(now).getFullYear()) + "-01-01+json"], | ||
| }, | ||
| }).do_request().as(resp, (resp.StatusCode == 200) ? | ||
| bytes(resp.Body).decode_json().as(body, | ||
| { | ||
| "host_list": body.results.map(e, state.url.trim_right("/") + "/api/atlas/v2/groups/" + state.group_id + "/processes/" + e.id + "/disks/"), | ||
| "next": 0, | ||
| "page_num": body.links.exists_one(res, res.rel == "next") ? (int(state.page_num) + 1) : 1, | ||
| }) | ||
| : | ||
| { | ||
| "events": { | ||
| "error": { | ||
| "code": string(resp.StatusCode), | ||
| "id": string(resp.Status), | ||
| "message": "GET:" + | ||
| ( | ||
| (size(resp.Body) != 0) ? | ||
| string(resp.Body) | ||
| : | ||
| string(resp.Status) + " (" + string(resp.StatusCode) + ")" | ||
| ), | ||
| }, | ||
| }, | ||
| "want_more": false, | ||
| } | ||
| ) | ||
| ) | ||
| ).as(state, (state.next >= size(state.host_list)) ? {} : | ||
| ( | ||
| (has(state.disk_list) && size(state.disk_list) > 0) ? | ||
| state | ||
| : | ||
| state.with( | ||
| request("GET", string(state.host_list[state.next] + "?pageNum=" + string(state.disk_page_num) + "&itemsPerPage=100")) | ||
| .with({ | ||
| "Header": { | ||
| "Accept": ["application/vnd.atlas." + string(timestamp(now).getFullYear()) + "-01-01+json"], | ||
| }, | ||
| }).do_request().as(resp, (resp.StatusCode == 200) ? | ||
| bytes(resp.Body).decode_json().as(body, | ||
| { | ||
| "disk_list": body.results.map(e, e.partitionName), | ||
| "disk_next": 0, | ||
| "disk_page_num": body.links.exists_one(res, res.rel == "next") ? (int(state.disk_page_num) + 1) : 1, | ||
| } | ||
| ) | ||
| : | ||
| { | ||
| "events": { | ||
| "error": { | ||
| "code": string(resp.StatusCode), | ||
| "id": string(resp.Status), | ||
| "message": "GET:" + | ||
| ( | ||
| (size(resp.Body) != 0) ? | ||
| string(resp.Body) | ||
| : | ||
| string(resp.Status) + " (" + string(resp.StatusCode) + ")" | ||
| ), | ||
| }, | ||
| }, | ||
| "want_more": false, | ||
| } | ||
| ) | ||
| ) | ||
| ).as(state, (state.disk_next >= size(state.disk_list)) ? {} : | ||
| request("GET", string(state.host_list[state.next] + state.disk_list[state.disk_next] + state.query)) | ||
| .with({ | ||
| "Header": { | ||
| "Accept": ["application/vnd.atlas." + string(timestamp(now).getFullYear()) + "-01-01+json"], | ||
| }, | ||
| }).do_request().as(res, (res.StatusCode == 200) ? | ||
| { | ||
| "events": bytes(res.Body).decode_json().as(f, | ||
| f.with( | ||
| { | ||
| "response": zip( | ||
| // Combining measurement names and actual values of measurement to generate `key : value` pairs. | ||
| f.measurements.map(m, m.name), | ||
| f.measurements.map(m, m.dataPoints.map(d, d.value).as(v, (size(v) == 0) ? null : (v[0]))) | ||
| ), | ||
| } | ||
| ).drop(["measurements", "links"]) | ||
| ), | ||
| "disk_list": (int(state.disk_next) + 1 < size(state.disk_list)) ? state.disk_list : [], | ||
| "disk_next": (int(state.disk_next) + 1 < size(state.disk_list)) ? (int(state.disk_next) + 1) : 0, | ||
| "disk_page_num": state.disk_page_num, | ||
| "host_list": (int(state.next) + 1 >= size(state.host_list) && int(state.disk_page_num) == 1 && int(state.disk_next) + 1 >= size(state.disk_list)) ? [] : state.host_list, | ||
| "next": (int(state.disk_next) + 1 >= size(state.disk_list) && int(state.disk_page_num) == 1 && int(state.next) + 1 < size(state.host_list)) ? (int(state.next) + 1) : int(state.next), | ||
| "want_more": int(state.next) + 1 < size(state.host_list) || int(state.page_num) != 1 || int(state.disk_next) + 1 < size(state.disk_list) || int(state.disk_page_num) != 1, | ||
| "page_num": state.page_num, | ||
| "group_id": state.group_id, | ||
| "query": state.query, | ||
| } | ||
| : | ||
| { | ||
| "events": { | ||
| "error": { | ||
| "code": string(res.StatusCode), | ||
| "id": string(res.Status), | ||
| "message": "GET:" + | ||
| ( | ||
| (size(res.Body) != 0) ? | ||
| string(res.Body) | ||
| : | ||
| string(res.Status) + " (" + string(res.StatusCode) + ")" | ||
| ), | ||
| }, | ||
| }, | ||
| "want_more": false, | ||
| } | ||
| ) | ||
| ) | ||
| ) |
Oops, something went wrong.