From 2434b86f2d8a06ba4cd5898017ce5aa95f9c6e4c Mon Sep 17 00:00:00 2001 From: Max Leske Date: Mon, 13 Nov 2023 10:18:20 +0100 Subject: [PATCH] feat: improve schema - deprecate some fields in favor of new ones - add `retry_once` option --- spec/v1.1/ftw.md | 458 ++++++++++++++++++++++++++++++----------- types/examples.go | 20 +- types/overrides_doc.go | 40 ++-- types/test_doc.go | 105 +++++++--- types/types.go | 134 ++++++++---- types/types_test.go | 109 +++++----- 6 files changed, 602 insertions(+), 264 deletions(-) diff --git a/spec/v1.1/ftw.md b/spec/v1.1/ftw.md index 209981b..c4fa341 100644 --- a/spec/v1.1/ftw.md +++ b/spec/v1.1/ftw.md @@ -46,8 +46,37 @@ Examples: ```yaml -# Tests -tests: the tests +tests: + - test_title: 123456-1 + rule_id: 0 + test_id: 0 + desc: Unix RCE using `time` + stages: + - description: Get cookie from server + input: + dest_addr: 192.168.0.1 + port: 8080 + protocol: http + uri: /test + version: HTTP/1.1 + method: REPORT + headers: + Accept: '*/*' + Host: localhost + User-Agent: CRS Tests + save_cookie: false + stop_magic: true + autocomplete_headers: false + encoded_request: TXkgRGF0YQo= + output: + status: 200 + log_contains: nothing + log: + expect_id: 123456 + no_expect_id: 123456 + match_regex: id[:\s"]*123456 + no_match_regex: id[:\s"]*123456 + expect_error: true ``` @@ -205,8 +234,36 @@ Appears in: ```yaml -# Tests -the tests +- test_title: 123456-1 + rule_id: 0 + test_id: 0 + desc: Unix RCE using `time` + stages: + - description: Get cookie from server + input: + dest_addr: 192.168.0.1 + port: 8080 + protocol: http + uri: /test + version: HTTP/1.1 + method: REPORT + headers: + Accept: '*/*' + Host: localhost + User-Agent: CRS Tests + save_cookie: false + stop_magic: true + autocomplete_headers: false + encoded_request: TXkgRGF0YQo= + output: + status: 200 + log_contains: nothing + log: + expect_id: 123456 + no_expect_id: 123456 + match_regex: id[:\s"]*123456 + no_match_regex: id[:\s"]*123456 + expect_error: true ``` @@ -220,7 +277,7 @@ the tests
-TestTitle the title of this particular test. It is used for inclusion/exclusion of each run by the tool. +TestTitle is the title of this particular test. It is used for inclusion/exclusion of each run by the tool. @@ -228,8 +285,7 @@ Examples: ```yaml -# TestTitle -test_title: 920100-1 +test_title: 123456-1 ``` @@ -239,13 +295,12 @@ test_title: 920100-1
-desc string +rule_id int
-TestDescription is the description for this particular test. Should be used to describe the internals of -the specific things this test is targeting. +RuleId is the ID of the rule this test targets @@ -253,8 +308,8 @@ Examples: ```yaml -# TestDescription -desc: This test targets something +# RuleId +rule_id: 123456 ``` @@ -264,12 +319,12 @@ desc: This test targets something
-stages []Stage +test_id int
-Stages is the list of all the stages to perform this test. +TestId is the ID of the test, in relation to `rule_id` @@ -277,33 +332,8 @@ Examples: ```yaml -# Stages -stages: - - stage: - input: - dest_addr: 192.168.0.1 - port: 8080 - protocol: http - uri: /test - version: HTTP/1.1 - method: REPORT - headers: - Accept: '*/*' - Host: localhost - User-Agent: CRS Tests - save_cookie: false - stop_magic: true - autocomplete_headers: false - encoded_request: TXkgRGF0YQo= - output: - status: 200 - log_contains: nothing - log: - expect_id: 123456 - expect_id: 123456 - match_regex: id[:\s"]*123456 - no_match_regex: id[:\s"]*123456 - expect_error: true +# TestId +test_id: 4 ``` @@ -311,59 +341,38 @@ stages:
+
+desc string +
+
+TestDescription is the description for this particular test. Should be used to describe the internals of +the specific things this test is targeting. -## Stage - -Appears in: -- Test.stages +Examples: ```yaml -# Stages -- stage: - input: - dest_addr: 192.168.0.1 - port: 8080 - protocol: http - uri: /test - version: HTTP/1.1 - method: REPORT - headers: - Accept: '*/*' - Host: localhost - User-Agent: CRS Tests - save_cookie: false - stop_magic: true - autocomplete_headers: false - encoded_request: TXkgRGF0YQo= - output: - status: 200 - log_contains: nothing - log: - expect_id: 123456 - expect_id: 123456 - match_regex: id[:\s"]*123456 - no_match_regex: id[:\s"]*123456 - expect_error: true +desc: Unix RCE using `time` ``` +
-stage StageData +stages []Stage
-StageData is an individual test stage +Stages is the list of all the stages to perform this test. @@ -371,9 +380,9 @@ Examples: ```yaml -# StageData -stage: - input: +stages: + - description: Get cookie from server + input: dest_addr: 192.168.0.1 port: 8080 protocol: http @@ -388,12 +397,12 @@ stage: stop_magic: true autocomplete_headers: false encoded_request: TXkgRGF0YQo= - output: + output: status: 200 log_contains: nothing log: expect_id: 123456 - expect_id: 123456 + no_expect_id: 123456 match_regex: id[:\s"]*123456 no_match_regex: id[:\s"]*123456 expect_error: true @@ -408,16 +417,98 @@ stage: -## StageData +## Stage Appears in: -- Stage.stage +- Test.stages + + +```yaml +- description: Get cookie from server + input: + dest_addr: 192.168.0.1 + port: 8080 + protocol: http + uri: /test + version: HTTP/1.1 + method: REPORT + headers: + Accept: '*/*' + Host: localhost + User-Agent: CRS Tests + save_cookie: false + stop_magic: true + autocomplete_headers: false + encoded_request: TXkgRGF0YQo= + output: + status: 200 + log_contains: nothing + log: + expect_id: 123456 + no_expect_id: 123456 + match_regex: id[:\s"]*123456 + no_match_regex: id[:\s"]*123456 + expect_error: true +``` + + + +
+ +
+ +stage StageData + +
+
+ +StageData is an individual test stage. + +
+ +
+ +
+ +description string + +
+
+ +Describes the purpose of this stage. + + + +Examples: ```yaml -# StageData +description: Get cookie from server +``` + + +
+ +
+ +
+ +input Input + +
+
+ +Input is the data that is passed to the test + + + +Examples: + + +```yaml +# Input input: dest_addr: 192.168.0.1 port: 8080 @@ -433,18 +524,59 @@ input: stop_magic: true autocomplete_headers: false encoded_request: TXkgRGF0YQo= +``` + + +
+ +
+ +
+ +output Output + +
+
+ +Output is the data that is returned from the test + + + +Examples: + + +```yaml +# Output output: status: 200 log_contains: nothing log: expect_id: 123456 - expect_id: 123456 + no_expect_id: 123456 match_regex: id[:\s"]*123456 no_match_regex: id[:\s"]*123456 expect_error: true ``` +
+ +
+ + + + + +## StageData + +Appears in: + + +- Stage.stage + + + +
@@ -507,7 +639,7 @@ output: log_contains: nothing log: expect_id: 123456 - expect_id: 123456 + no_expect_id: 123456 match_regex: id[:\s"]*123456 no_match_regex: id[:\s"]*123456 expect_error: true @@ -527,6 +659,8 @@ output: Appears in: +- Stage.input + - StageData.input @@ -547,6 +681,23 @@ stop_magic: true autocomplete_headers: false encoded_request: TXkgRGF0YQo= ``` +```yaml +# Input +dest_addr: 192.168.0.1 +port: 8080 +protocol: http +uri: /test +version: HTTP/1.1 +method: REPORT +headers: + Accept: '*/*' + Host: localhost + User-Agent: CRS Tests +save_cookie: false +stop_magic: true +autocomplete_headers: false +encoded_request: TXkgRGF0YQo= +``` @@ -878,6 +1029,8 @@ raw_request: TXkgRGF0YQo= Appears in: +- Stage.output + - StageData.output @@ -887,7 +1040,18 @@ status: 200 log_contains: nothing log: expect_id: 123456 + no_expect_id: 123456 + match_regex: id[:\s"]*123456 + no_match_regex: id[:\s"]*123456 +expect_error: true +``` +```yaml +# Output +status: 200 +log_contains: nothing +log: expect_id: 123456 + no_expect_id: 123456 match_regex: id[:\s"]*123456 no_match_regex: id[:\s"]*123456 expect_error: true @@ -1010,7 +1174,7 @@ Examples: ```yaml log: expect_id: 123456 - expect_id: 123456 + no_expect_id: 123456 match_regex: id[:\s"]*123456 no_match_regex: id[:\s"]*123456 ``` @@ -1058,7 +1222,7 @@ Appears in: ```yaml expect_id: 123456 -expect_id: 123456 +no_expect_id: 123456 match_regex: id[:\s"]*123456 no_match_regex: id[:\s"]*123456 ``` @@ -1074,10 +1238,17 @@ no_match_regex: id[:\s"]*123456
-description: | - Expect the given ID to be contained in the log output. - examples: - - exampleLog.ExpectId +Expect the given ID to be contained in the log output. + + + +Examples: + + +```yaml +expect_id: 123456 +``` +
@@ -1085,15 +1256,22 @@ description: |
-expect_id int +no_expect_id int
-description: | - Expect the given ID _not_ to be contained in the log output. - examples: - - exampleLog.NoExpectId +Expect the given ID _not_ to be contained in the log output. + + + +Examples: + + +```yaml +no_expect_id: 123456 +``` +
@@ -1230,9 +1408,7 @@ Examples: ```yaml test_overrides: - rule_id: 920100 - test_ids: - - 4 - - 6 + test_ids: [4, 6] reason: |- nginx returns 400 when `Content-Length` header is sent in a `Transfer-Encoding: chunked` request. @@ -1242,7 +1418,7 @@ test_overrides: log_contains: nothing log: expect_id: 123456 - expect_id: 123456 + no_expect_id: 123456 match_regex: id[:\s"]*123456 no_match_regex: id[:\s"]*123456 expect_error: true @@ -1362,9 +1538,7 @@ Appears in: ```yaml - rule_id: 920100 - test_ids: - - 4 - - 6 + test_ids: [4, 6] reason: |- nginx returns 400 when `Content-Length` header is sent in a `Transfer-Encoding: chunked` request. @@ -1374,7 +1548,7 @@ Appears in: log_contains: nothing log: expect_id: 123456 - expect_id: 123456 + no_expect_id: 123456 match_regex: id[:\s"]*123456 no_match_regex: id[:\s"]*123456 expect_error: true @@ -1414,11 +1588,20 @@ rule_id: 920100
-description: | - IDs of the tests for rule_id that overrides should be applied to. - If this field is not set, the overrides will be applied to all tests of rule_id. - examples: - - value: [5, 6] +IDs of the tests for rule_id that overrides should be applied to. +If this field is not set, the overrides will be applied to all tests of rule_id. + + + +Examples: + + +```yaml +test_ids: + - 4 + - 6 +``` +
@@ -1469,6 +1652,31 @@ expect_failure: true ``` +
+ +
+ +
+ +retry_once bool + +
+
+ +Whether a stage should be retried once in case of failure. +This option is primarily a workaround for a race condition in phase 5, +where the log entry of a rule may be flushed after the test end marker. + + + +Examples: + + +```yaml +retry_once: true +``` + +
@@ -1629,7 +1837,7 @@ Examples: ```yaml log: expect_id: 123456 - expect_id: 123456 + no_expect_id: 123456 match_regex: id[:\s"]*123456 no_match_regex: id[:\s"]*123456 ``` @@ -1677,7 +1885,7 @@ Appears in: ```yaml expect_id: 123456 -expect_id: 123456 +no_expect_id: 123456 match_regex: id[:\s"]*123456 no_match_regex: id[:\s"]*123456 ``` @@ -1693,10 +1901,17 @@ no_match_regex: id[:\s"]*123456
-description: | - Expect the given ID to be contained in the log output. - examples: - - exampleLog.ExpectId +Expect the given ID to be contained in the log output. + + + +Examples: + + +```yaml +expect_id: 123456 +``` +
@@ -1704,15 +1919,22 @@ description: |
-expect_id int +no_expect_id int
-description: | - Expect the given ID _not_ to be contained in the log output. - examples: - - exampleLog.NoExpectId +Expect the given ID _not_ to be contained in the log output. + + + +Examples: + + +```yaml +no_expect_id: 123456 +``` +
diff --git a/types/examples.go b/types/examples.go index 766b74a..cf7eff6 100644 --- a/types/examples.go +++ b/types/examples.go @@ -4,14 +4,21 @@ package types var ( - exampleStageData = StageData{ - Input: exampleInput, - Output: exampleOutput, + exampleTests = []Test{ + exampleTest, + } + exampleTest = Test{ + TestTitle: "123456-1", + TestDescription: "Unix RCE using `time`", + Stages: exampleStages, + } + exampleStage = Stage{ + Description: "Get cookie from server", + Input: exampleInput, + Output: exampleOutput, } exampleStages = []Stage{ - { - exampleStageData, - }, + exampleStage, } exampleHeaders = map[string]string{ "User-Agent": "CRS Tests", @@ -40,7 +47,6 @@ var ( Log: exampleLog, ExpectError: boolPtr(true), } - exampleLog = Log{ ExpectId: 123456, NoExpectId: 123456, diff --git a/types/overrides_doc.go b/types/overrides_doc.go index a328b1e..b766c18 100644 --- a/types/overrides_doc.go +++ b/types/overrides_doc.go @@ -88,20 +88,21 @@ func init() { FieldName: "test_overrides", }, } - TestOverrideDoc.Fields = make([]encoder.Doc, 5) + TestOverrideDoc.Fields = make([]encoder.Doc, 6) TestOverrideDoc.Fields[0].Name = "rule_id" TestOverrideDoc.Fields[0].Type = "int" TestOverrideDoc.Fields[0].Note = "" TestOverrideDoc.Fields[0].Description = "ID of the rule this test targets." TestOverrideDoc.Fields[0].Comments[encoder.LineComment] = "ID of the rule this test targets." - TestOverrideDoc.Fields[0].AddExample("", 920100) + TestOverrideDoc.Fields[0].AddExample("", testOverridesExample[0].RuleId) TestOverrideDoc.Fields[1].Name = "test_ids" TestOverrideDoc.Fields[1].Type = "[]int" TestOverrideDoc.Fields[1].Note = "" - TestOverrideDoc.Fields[1].Description = "description: |\n IDs of the tests for rule_id that overrides should be applied to.\n If this field is not set, the overrides will be applied to all tests of rule_id.\n examples:\n - value: [5, 6]" - TestOverrideDoc.Fields[1].Comments[encoder.LineComment] = " description: |" + TestOverrideDoc.Fields[1].Description = "IDs of the tests for rule_id that overrides should be applied to.\nIf this field is not set, the overrides will be applied to all tests of rule_id." + TestOverrideDoc.Fields[1].Comments[encoder.LineComment] = "IDs of the tests for rule_id that overrides should be applied to." + TestOverrideDoc.Fields[1].AddExample("", testOverridesExample[0].TestIds) TestOverrideDoc.Fields[2].Name = "reason" TestOverrideDoc.Fields[2].Type = "string" TestOverrideDoc.Fields[2].Note = "" @@ -116,13 +117,20 @@ func init() { TestOverrideDoc.Fields[3].Comments[encoder.LineComment] = "Whether this test is expected to fail for this particular configuration." TestOverrideDoc.Fields[3].AddExample("", true) - TestOverrideDoc.Fields[4].Name = "output" - TestOverrideDoc.Fields[4].Type = "Output" + TestOverrideDoc.Fields[4].Name = "retry_once" + TestOverrideDoc.Fields[4].Type = "bool" TestOverrideDoc.Fields[4].Note = "" - TestOverrideDoc.Fields[4].Description = "Specifies overrides on the test output" - TestOverrideDoc.Fields[4].Comments[encoder.LineComment] = "Specifies overrides on the test output" + TestOverrideDoc.Fields[4].Description = "Whether a stage should be retried once in case of failure.\nThis option is primarily a workaround for a race condition in phase 5,\nwhere the log entry of a rule may be flushed after the test end marker." + TestOverrideDoc.Fields[4].Comments[encoder.LineComment] = "Whether a stage should be retried once in case of failure." - TestOverrideDoc.Fields[4].AddExample("", 400) + TestOverrideDoc.Fields[4].AddExample("", true) + TestOverrideDoc.Fields[5].Name = "output" + TestOverrideDoc.Fields[5].Type = "Output" + TestOverrideDoc.Fields[5].Note = "" + TestOverrideDoc.Fields[5].Description = "Specifies overrides on the test output" + TestOverrideDoc.Fields[5].Comments[encoder.LineComment] = "Specifies overrides on the test output" + + TestOverrideDoc.Fields[5].AddExample("", 400) OutputDoc.Type = "Output" OutputDoc.Comments[encoder.LineComment] = "" @@ -194,13 +202,17 @@ func init() { LogDoc.Fields[0].Name = "expect_id" LogDoc.Fields[0].Type = "int" LogDoc.Fields[0].Note = "" - LogDoc.Fields[0].Description = "description: |\n Expect the given ID to be contained in the log output.\n examples:\n - exampleLog.ExpectId" - LogDoc.Fields[0].Comments[encoder.LineComment] = " description: |" - LogDoc.Fields[1].Name = "expect_id" + LogDoc.Fields[0].Description = "Expect the given ID to be contained in the log output." + LogDoc.Fields[0].Comments[encoder.LineComment] = "Expect the given ID to be contained in the log output." + + LogDoc.Fields[0].AddExample("", exampleLog.ExpectId) + LogDoc.Fields[1].Name = "no_expect_id" LogDoc.Fields[1].Type = "int" LogDoc.Fields[1].Note = "" - LogDoc.Fields[1].Description = "description: |\n Expect the given ID _not_ to be contained in the log output.\n examples:\n - exampleLog.NoExpectId" - LogDoc.Fields[1].Comments[encoder.LineComment] = " description: |" + LogDoc.Fields[1].Description = "Expect the given ID _not_ to be contained in the log output." + LogDoc.Fields[1].Comments[encoder.LineComment] = "Expect the given ID _not_ to be contained in the log output." + + LogDoc.Fields[1].AddExample("", exampleLog.NoExpectId) LogDoc.Fields[2].Name = "match_regex" LogDoc.Fields[2].Type = "string" LogDoc.Fields[2].Note = "" diff --git a/types/test_doc.go b/types/test_doc.go index 48a3c30..fbdcac3 100644 --- a/types/test_doc.go +++ b/types/test_doc.go @@ -35,7 +35,7 @@ func init() { FTWTestDoc.Fields[1].Description = "Tests is a list of FTW tests" FTWTestDoc.Fields[1].Comments[encoder.LineComment] = "Tests is a list of FTW tests" - FTWTestDoc.Fields[1].AddExample("Tests", "the tests") + FTWTestDoc.Fields[1].AddExample("", exampleTests) FTWTestMetaDoc.Type = "FTWTestMeta" FTWTestMetaDoc.Comments[encoder.LineComment] = "" @@ -87,61 +87,92 @@ func init() { TestDoc.Comments[encoder.LineComment] = "" TestDoc.Description = "" - TestDoc.AddExample("Tests", "the tests") + TestDoc.AddExample("", exampleTests) TestDoc.AppearsIn = []encoder.Appearance{ { TypeName: "FTWTest", FieldName: "tests", }, } - TestDoc.Fields = make([]encoder.Doc, 3) + TestDoc.Fields = make([]encoder.Doc, 5) TestDoc.Fields[0].Name = "test_title" TestDoc.Fields[0].Type = "string" TestDoc.Fields[0].Note = "" - TestDoc.Fields[0].Description = "TestTitle the title of this particular test. It is used for inclusion/exclusion of each run by the tool." - TestDoc.Fields[0].Comments[encoder.LineComment] = "TestTitle the title of this particular test. It is used for inclusion/exclusion of each run by the tool." + TestDoc.Fields[0].Description = "TestTitle is the title of this particular test. It is used for inclusion/exclusion of each run by the tool." + TestDoc.Fields[0].Comments[encoder.LineComment] = "TestTitle is the title of this particular test. It is used for inclusion/exclusion of each run by the tool." - TestDoc.Fields[0].AddExample("TestTitle", "920100-1") - TestDoc.Fields[1].Name = "desc" - TestDoc.Fields[1].Type = "string" + TestDoc.Fields[0].AddExample("", exampleTest.TestTitle) + TestDoc.Fields[1].Name = "rule_id" + TestDoc.Fields[1].Type = "int" TestDoc.Fields[1].Note = "" - TestDoc.Fields[1].Description = "TestDescription is the description for this particular test. Should be used to describe the internals of\nthe specific things this test is targeting." - TestDoc.Fields[1].Comments[encoder.LineComment] = "TestDescription is the description for this particular test. Should be used to describe the internals of" + TestDoc.Fields[1].Description = "RuleId is the ID of the rule this test targets" + TestDoc.Fields[1].Comments[encoder.LineComment] = "RuleId is the ID of the rule this test targets" - TestDoc.Fields[1].AddExample("TestDescription", "This test targets something") - TestDoc.Fields[2].Name = "stages" - TestDoc.Fields[2].Type = "[]Stage" + TestDoc.Fields[1].AddExample("RuleId", 123456) + TestDoc.Fields[2].Name = "test_id" + TestDoc.Fields[2].Type = "int" TestDoc.Fields[2].Note = "" - TestDoc.Fields[2].Description = "Stages is the list of all the stages to perform this test." - TestDoc.Fields[2].Comments[encoder.LineComment] = "Stages is the list of all the stages to perform this test." + TestDoc.Fields[2].Description = "TestId is the ID of the test, in relation to `rule_id`" + TestDoc.Fields[2].Comments[encoder.LineComment] = "TestId is the ID of the test, in relation to `rule_id`" - TestDoc.Fields[2].AddExample("Stages", exampleStages) + TestDoc.Fields[2].AddExample("TestId", 4) + TestDoc.Fields[3].Name = "desc" + TestDoc.Fields[3].Type = "string" + TestDoc.Fields[3].Note = "" + TestDoc.Fields[3].Description = "TestDescription is the description for this particular test. Should be used to describe the internals of\nthe specific things this test is targeting." + TestDoc.Fields[3].Comments[encoder.LineComment] = "TestDescription is the description for this particular test. Should be used to describe the internals of" + + TestDoc.Fields[3].AddExample("", exampleTest.TestDescription) + TestDoc.Fields[4].Name = "stages" + TestDoc.Fields[4].Type = "[]Stage" + TestDoc.Fields[4].Note = "" + TestDoc.Fields[4].Description = "Stages is the list of all the stages to perform this test." + TestDoc.Fields[4].Comments[encoder.LineComment] = "Stages is the list of all the stages to perform this test." + + TestDoc.Fields[4].AddExample("", exampleStages) StageDoc.Type = "Stage" StageDoc.Comments[encoder.LineComment] = "" StageDoc.Description = "" - StageDoc.AddExample("Stages", exampleStages) + StageDoc.AddExample("", exampleStages) StageDoc.AppearsIn = []encoder.Appearance{ { TypeName: "Test", FieldName: "stages", }, } - StageDoc.Fields = make([]encoder.Doc, 1) + StageDoc.Fields = make([]encoder.Doc, 4) StageDoc.Fields[0].Name = "stage" StageDoc.Fields[0].Type = "StageData" StageDoc.Fields[0].Note = "" - StageDoc.Fields[0].Description = "StageData is an individual test stage" - StageDoc.Fields[0].Comments[encoder.LineComment] = "StageData is an individual test stage" - - StageDoc.Fields[0].AddExample("StageData", exampleStageData) + StageDoc.Fields[0].Description = "StageData is an individual test stage." + StageDoc.Fields[0].Comments[encoder.LineComment] = "StageData is an individual test stage." + StageDoc.Fields[1].Name = "description" + StageDoc.Fields[1].Type = "string" + StageDoc.Fields[1].Note = "" + StageDoc.Fields[1].Description = "Describes the purpose of this stage." + StageDoc.Fields[1].Comments[encoder.LineComment] = "Describes the purpose of this stage." + + StageDoc.Fields[1].AddExample("", exampleStage.Description) + StageDoc.Fields[2].Name = "input" + StageDoc.Fields[2].Type = "Input" + StageDoc.Fields[2].Note = "" + StageDoc.Fields[2].Description = "Input is the data that is passed to the test" + StageDoc.Fields[2].Comments[encoder.LineComment] = "Input is the data that is passed to the test" + + StageDoc.Fields[2].AddExample("Input", exampleInput) + StageDoc.Fields[3].Name = "output" + StageDoc.Fields[3].Type = "Output" + StageDoc.Fields[3].Note = "" + StageDoc.Fields[3].Description = "Output is the data that is returned from the test" + StageDoc.Fields[3].Comments[encoder.LineComment] = "Output is the data that is returned from the test" + + StageDoc.Fields[3].AddExample("Output", exampleOutput) StageDataDoc.Type = "StageData" StageDataDoc.Comments[encoder.LineComment] = "" StageDataDoc.Description = "" - - StageDataDoc.AddExample("StageData", exampleStageData) StageDataDoc.AppearsIn = []encoder.Appearance{ { TypeName: "Stage", @@ -168,8 +199,14 @@ func init() { InputDoc.Comments[encoder.LineComment] = "" InputDoc.Description = "" + InputDoc.AddExample("Input", exampleInput) + InputDoc.AddExample("Input", exampleInput) InputDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "Stage", + FieldName: "input", + }, { TypeName: "StageData", FieldName: "input", @@ -272,8 +309,14 @@ func init() { FTWTestOutputDoc.Comments[encoder.LineComment] = "" FTWTestOutputDoc.Description = "" + FTWTestOutputDoc.AddExample("Output", exampleOutput) + FTWTestOutputDoc.AddExample("Output", exampleOutput) FTWTestOutputDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "Stage", + FieldName: "output", + }, { TypeName: "StageData", FieldName: "output", @@ -338,13 +381,17 @@ func init() { FTWTestLogDoc.Fields[0].Name = "expect_id" FTWTestLogDoc.Fields[0].Type = "int" FTWTestLogDoc.Fields[0].Note = "" - FTWTestLogDoc.Fields[0].Description = "description: |\n Expect the given ID to be contained in the log output.\n examples:\n - exampleLog.ExpectId" - FTWTestLogDoc.Fields[0].Comments[encoder.LineComment] = " description: |" - FTWTestLogDoc.Fields[1].Name = "expect_id" + FTWTestLogDoc.Fields[0].Description = "Expect the given ID to be contained in the log output." + FTWTestLogDoc.Fields[0].Comments[encoder.LineComment] = "Expect the given ID to be contained in the log output." + + FTWTestLogDoc.Fields[0].AddExample("", exampleLog.ExpectId) + FTWTestLogDoc.Fields[1].Name = "no_expect_id" FTWTestLogDoc.Fields[1].Type = "int" FTWTestLogDoc.Fields[1].Note = "" - FTWTestLogDoc.Fields[1].Description = "description: |\n Expect the given ID _not_ to be contained in the log output.\n examples:\n - exampleLog.NoExpectId" - FTWTestLogDoc.Fields[1].Comments[encoder.LineComment] = " description: |" + FTWTestLogDoc.Fields[1].Description = "Expect the given ID _not_ to be contained in the log output." + FTWTestLogDoc.Fields[1].Comments[encoder.LineComment] = "Expect the given ID _not_ to be contained in the log output." + + FTWTestLogDoc.Fields[1].AddExample("", exampleLog.NoExpectId) FTWTestLogDoc.Fields[2].Name = "match_regex" FTWTestLogDoc.Fields[2].Type = "string" FTWTestLogDoc.Fields[2].Note = "" diff --git a/types/types.go b/types/types.go index 65fde7f..b3bf111 100644 --- a/types/types.go +++ b/types/types.go @@ -27,8 +27,7 @@ type FTWTest struct { // description: | // Tests is a list of FTW tests // examples: - // - name: Tests - // value: "\"the tests\"" + // - value: exampleTests Tests []Test `yaml:"tests"` } @@ -46,6 +45,8 @@ type FTWTestMeta struct { // examples: // - name: Enabled // value: false + // + // Deprecated: ignored; use platform specific overrides instead Enabled *bool `yaml:"enabled,omitempty"` // description: | @@ -73,39 +74,72 @@ type FTWTestMeta struct { // Test is an individual test. One test can have multiple stages. type Test struct { // description: | - // TestTitle the title of this particular test. It is used for inclusion/exclusion of each run by the tool. + // TestTitle is the title of this particular test. It is used for inclusion/exclusion of each run by the tool. + // examples: + // - value: exampleTest.TestTitle + // + // Deprecated: use `rule_id` and `test_id` + TestTitle string `yaml:"test_title,omitempty"` + + // description: | + // RuleId is the ID of the rule this test targets + // examples: + // - name: RuleId + // value: 123456 + RuleId int `yaml:"rule_id"` + + // description: | + // TestId is the ID of the test, in relation to `rule_id` // examples: - // - name: TestTitle - // value: "\"920100-1\"" - TestTitle string `yaml:"test_title"` + // - name: TestId + // value: 4 + TestId int `yaml:"test_id"` // description: | // TestDescription is the description for this particular test. Should be used to describe the internals of // the specific things this test is targeting. // examples: - // - name: TestDescription - // value: "\"This test targets something\"" + // - value: exampleTest.TestDescription TestDescription string `yaml:"desc,omitempty"` // description: | // Stages is the list of all the stages to perform this test. // examples: - // - name: Stages - // value: exampleStages + // - value: exampleStages Stages []Stage `yaml:"stages"` } // Stage is a list of stages type Stage struct { // description: | - // StageData is an individual test stage + // StageData is an individual test stage. + // + // Deprecated: use the other fields of `Stage` + SD StageData `yaml:"stage,omitempty"` + // description: | + // Describes the purpose of this stage. + // examples: + // - value: exampleStage.Description + Description string `yaml:"description,omitempty"` + + // description: | + // Input is the data that is passed to the test // examples: - // - name: StageData - // value: exampleStageData - SD StageData `yaml:"stage"` + // - name: Input + // value: exampleInput + Input Input `yaml:"input"` + + // description: | + // Output is the data that is returned from the test + // examples: + // - name: Output + // value: exampleOutput + Output Output `yaml:"output"` } // StageData is the data that is passed to the test, and the data that is returned from the test +// +// Deprecated: use the other fields of `stage` type StageData struct { // description: | // Input is the data that is passed to the test @@ -193,6 +227,8 @@ type Input struct { // examples: // - name: StopMagic // value: false + // + // Deprecated: use AutocompleteHeaders instead StopMagic *bool `yaml:"stop_magic" koanf:"stop_magic,omitempty"` // description: | @@ -216,6 +252,8 @@ type Input struct { // examples: // - name: RAWRequest // value: "\"TXkgRGF0YQo=\"" + // + // Deprecated: use `encoded_request` RAWRequest string `yaml:"raw_request,omitempty" koanf:"raw_request,omitempty"` } @@ -241,7 +279,7 @@ type Output struct { // - name: LogContains // value: "\"id 920100\"" // - // deprecated: use Log instead + // Deprecated: use Log instead LogContains string `yaml:"log_contains,omitempty"` // description: | @@ -250,7 +288,7 @@ type Output struct { // - name: NoLogContains // value: "\"id 920100\"" // - // deprecated: use Log instead + // Deprecated: use Log instead NoLogContains string `yaml:"no_log_contains,omitempty"` // description: | @@ -272,14 +310,14 @@ type Log struct { // description: | // Expect the given ID to be contained in the log output. // examples: - // - exampleLog.ExpectId + // - value: exampleLog.ExpectId ExpectId int `yaml:"expect_id,omitempty"` // description: | // Expect the given ID _not_ to be contained in the log output. // examples: - // - exampleLog.NoExpectId - NoExpectId int `yaml:"expect_id,omitempty"` + // - value: exampleLog.NoExpectId + NoExpectId int `yaml:"no_expect_id,omitempty"` // description: | // Expect the regular expression to match log content for the current test. @@ -297,76 +335,84 @@ type Log struct { // FTWOverrides describes platform specific overrides for tests type FTWOverrides struct { // description: | - // The version field designates the version of the schema that validates this file + // The version field designates the version of the schema that validates this file // examples: - // - value: "\"v0.1.0\"" + // - value: "\"v0.1.0\"" Version string `yaml:"version"` // description: | - // Meta describes the metadata information + // Meta describes the metadata information // examples: - // - value: metaExample + // - value: metaExample Meta FTWOverridesMeta `yaml:"meta"` // description: | - // List of test override specifications + // List of test override specifications // examples: - // - value: testOverridesExample + // - value: testOverridesExample TestOverrides []TestOverride `yaml:"test_overrides"` } // FTWOverridesMeta describes the metadata information of this yaml file type FTWOverridesMeta struct { // description: | - // The name of the WAF engine the tests are expected to run against + // The name of the WAF engine the tests are expected to run against // examples: - // - value: "\"coraza\"" + // - value: "\"coraza\"" Engine string `yaml:"engine"` // description: | - // The name of the platform (e.g., web server) the tests are expected to run against + // The name of the platform (e.g., web server) the tests are expected to run against // examples: - // - value: "\"nginx\"" + // - value: "\"nginx\"" Platform string `yaml:"platform"` // description: | - // Custom annotations; can be used to add additional meta information + // Custom annotations; can be used to add additional meta information // examples: - // - value: annotationsExample + // - value: annotationsExample Annotations map[string]string `yaml:"annotations"` } // TestOverride describes overrides for a single test type TestOverride struct { // description: | - // ID of the rule this test targets. + // ID of the rule this test targets. // examples: - // - value: 920100 + // - value: testOverridesExample[0].RuleId RuleId int `yaml:"rule_id"` // description: | - // IDs of the tests for rule_id that overrides should be applied to. - // If this field is not set, the overrides will be applied to all tests of rule_id. + // IDs of the tests for rule_id that overrides should be applied to. + // If this field is not set, the overrides will be applied to all tests of rule_id. // examples: - // - value: [5, 6] - TestIds []int `yaml:"test_ids,omitempty"` + // - value: testOverridesExample[0].TestIds + TestIds []int `yaml:"test_ids,flow,omitempty"` // description: | - // Describes why this override is necessary. + // Describes why this override is necessary. // examples: - // - value: reasonExample + // - value: reasonExample Reason string `yaml:"reason"` // description: | - // Whether this test is expected to fail for this particular configuration. - // Default: false + // Whether this test is expected to fail for this particular configuration. + // Default: false // examples: - // - value: true + // - value: true ExpectFailure *bool `yaml:"expect_failure,omitempty"` // description: | - // Specifies overrides on the test output + // Whether a stage should be retried once in case of failure. + // This option is primarily a workaround for a race condition in phase 5, + // where the log entry of a rule may be flushed after the test end marker. + // examples: + // - value: true + RetryOnce *bool `yaml:"retry_once,omitempty"` + + // description: | + // Specifies overrides on the test output // examples: - // - value: 400 + // - value: 400 Output Output `yaml:"output"` } diff --git a/types/types_test.go b/types/types_test.go index f88cde4..d6d9fd9 100644 --- a/types/types_test.go +++ b/types/types_test.go @@ -19,29 +19,40 @@ meta: description: "Simple YAML to test that the schema is working." tests: - test_title: 1234-1 + rule_id: 1234 + test_id: 1 desc: "Test that the schema is working." stages: - - stage: - input: - dest_addr: "127.0.0.1" - port: 80 - headers: - User-Agent: "FTW Schema Tests" - Host: "localhost" - output: - no_log_contains: 'id "1234"' + - input: + dest_addr: "192.168.0.1" + port: 8080 + method: "REPORT" + headers: + User-Agent: "CRS Tests" + Host: "localhost" + Accept: "*/*" + encoded_request: "TXkgRGF0YQo=" + uri: "/test" + protocol: "http" + autocomplete_headers: false + stop_magic: true + save_cookie: false + output: + status: 200 + response_contains: "" + log_contains: "nothing" + no_log_contains: "" - test_title: 1234-2 stages: - - stage: - input: - dest_addr: "127.0.0.1" - port: 80 - method: "OPTIONS" - headers: - User-Agent: "FTW Schema Tests" - Host: "localhost" - output: - status: 200 + - input: + dest_addr: "127.0.0.1" + port: 80 + method: "OPTIONS" + headers: + User-Agent: "FTW Schema Tests" + Host: "localhost" + output: + status: 200 ` var ftwTest = &FTWTest{ @@ -55,22 +66,14 @@ var ftwTest = &FTWTest{ Tests: []Test{ { TestTitle: "1234-1", + RuleId: 1234, + TestId: 1, TestDescription: "Test that the schema is working.", Stages: []Stage{ { - SD: StageData{ - Input: Input{ - DestAddr: strPtr("127.0.0.1"), - Port: intPtr(80), - Headers: map[string]string{ - "User-Agent": "FTW Schema Tests", - "Host": "localhost", - }, - }, - Output: Output{ - NoLogContains: "id \"1234\"", - }, - }, + Description: exampleStage.Description, + Input: exampleInput, + Output: exampleOutput, }, }, }, @@ -78,20 +81,18 @@ var ftwTest = &FTWTest{ TestTitle: "1234-2", Stages: []Stage{ { - SD: StageData{ - Input: Input{ - DestAddr: strPtr("127.0.0.1"), - Port: intPtr(80), - Method: strPtr("OPTIONS"), - Headers: map[string]string{ - "User-Agent": "FTW Schema Tests", - "Host": "localhost", - }, - }, - Output: Output{ - Status: 200, + Input: Input{ + DestAddr: strPtr("127.0.0.1"), + Port: intPtr(80), + Method: strPtr("OPTIONS"), + Headers: map[string]string{ + "User-Agent": "FTW Schema Tests", + "Host": "localhost", }, }, + Output: Output{ + Status: 200, + }, }, }, }, @@ -115,22 +116,24 @@ func TestUnmarshalFTWTest(t *testing.T) { for i, test := range ftw.Tests { expectedTest := ftwTest.Tests[i] assert.Equal(expectedTest.TestTitle, test.TestTitle) + assert.Equal(expectedTest.RuleId, test.RuleId) + assert.Equal(expectedTest.TestId, test.TestId) assert.Len(test.Stages, len(expectedTest.Stages)) for j, stage := range test.Stages { expectedStage := expectedTest.Stages[j] - assert.Equal(expectedStage.SD.Input.DestAddr, stage.SD.Input.DestAddr) - assert.Equal(expectedStage.SD.Input.Port, stage.SD.Input.Port) - assert.Equal(expectedStage.SD.Input.Method, stage.SD.Input.Method) - assert.Len(stage.SD.Input.Headers, len(expectedStage.SD.Input.Headers)) + assert.Equal(expectedStage.Input.DestAddr, stage.Input.DestAddr) + assert.Equal(expectedStage.Input.Port, stage.Input.Port) + assert.Equal(expectedStage.Input.Method, stage.Input.Method) + assert.Len(stage.Input.Headers, len(expectedStage.Input.Headers)) - for k, header := range stage.SD.Input.Headers { - expectedHeader := expectedStage.SD.Input.Headers[k] + for k, header := range stage.Input.Headers { + expectedHeader := expectedStage.Input.Headers[k] assert.Equal(expectedHeader, header) } - assert.Equal(expectedStage.SD.Output.NoLogContains, stage.SD.Output.NoLogContains) - assert.Equal(expectedStage.SD.Output.Status, stage.SD.Output.Status) + assert.Equal(expectedStage.Output.NoLogContains, stage.Output.NoLogContains) + assert.Equal(expectedStage.Output.Status, stage.Output.Status) } } } @@ -156,6 +159,8 @@ test_overrides: status: 200 log_contains: "nothing" log: + expect_id: 123456 + no_expect_id: 123456 match_regex: 'id[:\s"]*123456' no_match_regex: 'id[:\s"]*123456' expect_error: true