From f2001a438ef6c64f41fc039b317d2eaa3a166c41 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 26 Jun 2023 13:04:09 -0400 Subject: [PATCH] Adds testcaserun skipped event Adds "skipped" as a new predicate for testcaserun events. A final outcome of "skip" is common for many test frameworks, so adding it as a possible outcome will improve interoperability. See further discussion in https://github.com/cdevents/spec/issues/140. Signed-off-by: Daniel Han --- examples/testcaserun_skipped.json | 31 ++++++ schemas/testcaserunskipped.json | 168 ++++++++++++++++++++++++++++++ testing-events.md | 27 ++++- 3 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 examples/testcaserun_skipped.json create mode 100644 schemas/testcaserunskipped.json diff --git a/examples/testcaserun_skipped.json b/examples/testcaserun_skipped.json new file mode 100644 index 00000000..ea678efb --- /dev/null +++ b/examples/testcaserun_skipped.json @@ -0,0 +1,31 @@ +{ + "context": { + "version": "0.4.0-draft", + "id": "271069a8-fc18-44f1-b38f-9d70a1695819", + "source": "/event/source/123", + "type": "dev.cdevents.testcaserun.skipped.0.1.0-draft", + "timestamp": "2023-03-20T14:27:05.315384Z" + }, + "subject": { + "id": "myTestCaseRun123", + "source": "/event/source/123", + "type": "testCaseRun", + "content": { + "reason": "Not running in this environment", + "environment": { + "id": "dev", + "source": "testkube-dev-123" + }, + "testSuiteRun": { + "id": "test-suite-111", + "source": "testkube-dev-123" + }, + "testCase": { + "id": "92834723894", + "version": "1.0", + "name": "Login Test", + "type": "integration" + } + } + } +} \ No newline at end of file diff --git a/schemas/testcaserunskipped.json b/schemas/testcaserunskipped.json new file mode 100644 index 00000000..94ad80dd --- /dev/null +++ b/schemas/testcaserunskipped.json @@ -0,0 +1,168 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://cdevents.dev/0.4.0-draft/schema/test-case-run-skipped-event", + "properties": { + "context": { + "properties": { + "version": { + "type": "string", + "minLength": 1 + }, + "id": { + "type": "string", + "minLength": 1 + }, + "source": { + "type": "string", + "minLength": 1 + }, + "type": { + "type": "string", + "enum": [ + "dev.cdevents.testcaserun.skipped.0.1.0-draft" + ], + "default": "dev.cdevents.testcaserun.skipped.0.1.0-draft" + }, + "timestamp": { + "type": "string", + "format": "date-time" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "version", + "id", + "source", + "type", + "timestamp" + ] + }, + "subject": { + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "source": { + "type": "string" + }, + "type": { + "type": "string", + "minLength": 1, + "enum": [ + "testCaseRun" + ], + "default": "testCaseRun" + }, + "content": { + "properties": { + "reason": { + "type": "string" + }, + "environment": { + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "source": { + "type": "string", + "minLength": 1, + "format": "uri-reference" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "id" + ] + }, + "testSuiteRun": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "source": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "id" + ] + }, + "testCase": { + "type": "object", + "additionalProperties": false, + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string" + }, + "name": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "performance", + "functional", + "unit", + "security", + "compliance", + "integration", + "e2e", + "other" + ] + }, + "uri": { + "type": "string", + "format": "uri" + } + } + } + }, + "additionalProperties": false, + "type": "object", + "required": [] + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "id", + "type", + "content" + ] + }, + "customData": { + "oneOf": [ + { + "type": "object" + }, + { + "type": "string", + "contentEncoding": "base64" + } + ] + }, + "customDataContentType": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "context", + "subject" + ] +} \ No newline at end of file diff --git a/testing-events.md b/testing-events.md index 00661908..2dfb63a1 100644 --- a/testing-events.md +++ b/testing-events.md @@ -18,9 +18,16 @@ Testing events covers the subjects and predicates related to test-execution perf This specification defines three subjects in this stage: `testCaseRun`, `testSuiteRun` and `testOutput`. +The predicates for `testCaseRun` must follow the following expectations: + +- The following events are valid initial events: `queued`, `started`, or `skipped`. +- The following events are valid terminal events that will *not* be followed by other events: `finished` and `skipped`. +- A `queued` event must be followed by one of the following: `started` or `finished`. +- A `started` event must be followed by a `finished` event. + | Subject | Description | Predicates | |---------------------------------|----------------------------------------------|------------------------------------------------------------------------------------------------------------| -| [`testCaseRun`](#testcaserun) | The execution of a software testCase | [`queued`](#testcaserun-queued), [`started`](#testcaserun-started), [`finished`](#testcaserun-finished) | +| [`testCaseRun`](#testcaserun) | The execution of a software testCase | [`queued`](#testcaserun-queued), [`started`](#testcaserun-started), [`finished`](#testcaserun-finished), [`skipped`](#testcaserun-skipped) | | [`testSuiteRun`](#testsuiterun) | The execution of a software testSuite | [`queued`](#testsuiterun-queued), [`started`](#testsuiterun-started), [`finished`](#testsuiterun-finished) | | [`testOutput`](#testoutput) | An output artifact produced by a testCaseRun | [`published`](#testoutput-published) | @@ -117,6 +124,24 @@ This event represents a finished testCase execution. The event will contain the | severity | `String (enum)` | Severity if the test failed, one of `low`, `medium`, `high`, `critical` | `critical` | | reason | `String` | A reason related to the outcome of the execution | `Canceled by user`, `Failed assertion`, `Timed out` | | +### [`testCaseRun skipped`](examples/testcaserun_skipped.json) + +This event represents a skipped testCaseRun execution. The event should only be emitted if there has been no prior "queued" or "started" event. + +- Event Type: __`dev.cdevents.testcaserun.skipped.0.1.0-draft`__ +- Predicate: skipped +- Subject: [`testCaseRun`](#testcaserun) + +| Field | Type | Description | Examples | Required | +|--------------|-----------------------------------------------------------------|----------------------------------------------------------------------------------|-------------------------------------------------------------------------|----------| +| id | `String` | Uniquely identifies the subject within the source. | `unitest-abc`, `e2e-test1`, `scan-image1` | ✅ | +| source | `URI-Reference` | [source](spec.md#source--context-) from the context | | | +| environment | `Object` [`environment`](continuous-deployment.md/#environment) | The environment in which this testCaseRun would have run, but was skipped. | `{"id": "1234"}`, `{"id": "dev", "source": "testkube-dev-123"}` | ✅ | +| testCase | `Object` [`testCase`](#testcase) | Definition of the testCase being executed | `{"id": "92834723894", "name": "Login Test", "type": "integration"}` | | +| testSuiteRun | `Object` [`testSuiteRun`](#testsuiterun) | A testSuiteRun to associate this testCaseRun with a containing testSuiteRun | `{"id":"Auth-TestSuite-execution-12334", "source": "staging/testkube"}` | | +| severity | `String` | Severity if the test failed, one of `low`, `medium`, `high`, `critical` | `critical` | +| reason | `String` | A reason for skipping the test case run. | `Not running in given environment`, `Skipping slow tests` | | + ### [`testSuiteRun queued`](examples/testsuiterun_queued.json) This event represents when a testSuiteRun has been queued for execution - and is waiting for applicable preconditions