From 7ecae6fb9a5c9bb07b46ee998bfd239b029a45b8 Mon Sep 17 00:00:00 2001 From: Matthew Schwartz Date: Fri, 24 May 2019 11:05:18 -0400 Subject: [PATCH 1/4] ODE-1264 Remove microseconds to allow second-level tolerance for timestamp comparisons --- odevalidator/validator.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/odevalidator/validator.py b/odevalidator/validator.py index cffe26d..3ca9ccc 100644 --- a/odevalidator/validator.py +++ b/odevalidator/validator.py @@ -48,7 +48,7 @@ def __init__(self, key, field_config = None, test_case = None): earliest_time = field_config.get('EarliestTime') if earliest_time is not None: try: - self.earliest_time = dateutil.parser.parse(earliest_time) + self.earliest_time = dateutil.parser.parse(earliest_time).replace(microsecond=0) except Exception as e: raise ValidatorException("Unable to parse configuration file timestamp EarliestTime for field %s=%s, error: %s" % (key, field_config, str(e))) latest_time = field_config.get('LatestTime') @@ -57,7 +57,7 @@ def __init__(self, key, field_config = None, test_case = None): self.latest_time = datetime.now(timezone.utc) else: try: - self.latest_time = dateutil.parser.parse(latest_time) + self.latest_time = dateutil.parser.parse(latest_time).replace(microsecond=0) except Exception as e: raise ValidatorException("Unable to parse configuration file timestamp LatestTime for field %s=%s, error: %s" % (key, field_config, str(e))) @@ -99,11 +99,11 @@ def _check_value(self, data_field_value, data): # check the value against it. Otherwise, carry on without a validation error validation = self._check_conditional(then_part, data_field_value, data) - # This is NOT a skipSequentialValidation condition and + # This is NOT a skipSequentialValidation condition and # therefore a field validation condition is met. If it is skipSequentialValidation - # we don't consider it a condition for field validation. Also, + # we don't consider it a condition for field validation. Also, field_validation_condition_met = True - + if not field_validation_condition_met: validation = self._check_unconditional(data_field_value) else: @@ -146,7 +146,7 @@ def _check_conditional(self, then_part, data_field_value, data): if data_field_value not in then_part['matchAgainst']: # the existing field value is not among the expected values validation = FieldValidationResult(False, "Value of Field ('%s') is not one of the expected values (%s)" % (data_field_value, then_part['matchAgainst']), self.path) - + return validation def _get_field_value(self, path_str, data): @@ -179,7 +179,7 @@ def _check_unconditional(self, data_field_value): return FieldValidationResult(False, "Value '%d' is less than lower limit '%d'" % (Decimal(data_field_value), self.lower_limit), self.path) elif self.type == TYPE_TIMESTAMP: try: - time_value = dateutil.parser.parse(data_field_value) + time_value = dateutil.parser.parse(data_field_value).replace(microsecond=0) if hasattr(self, 'earliest_time') and time_value < self.earliest_time: return FieldValidationResult(False, "Timestamp value '%s' occurs before earliest limit '%s'" % (time_value, self.earliest_time), self.path) @@ -193,9 +193,9 @@ def __str__(self): def to_json(self): return { - 'Path': self.path, - 'Type': self.type if hasattr(self, 'type') else None, - 'UpperLimit': self.upper_limit if hasattr(self, 'upper_limit') else None, + 'Path': self.path, + 'Type': self.type if hasattr(self, 'type') else None, + 'UpperLimit': self.upper_limit if hasattr(self, 'upper_limit') else None, 'LowerLimit': self.lower_limit if hasattr(self, 'lower_limit') else None, 'Values': self.values if hasattr(self, 'values') else None, 'EqualsValue': self.equals_value if hasattr(self, 'equals_value') else None, @@ -292,4 +292,4 @@ def check_headers(self, headers): for field in self.field_list: if not str.lower(field.path) == str.lower(csv_fields[index]): logger.warning("Warning: The data file CSV header '" + str.lower(csv_fields[index]) + "' does not match the config file field '" + str.lower(field.path) + "'"); - index += 1 \ No newline at end of file + index += 1 From 949130feab81db3d9f832b4abc38124e3e38f152 Mon Sep 17 00:00:00 2001 From: Matthew Schwartz Date: Fri, 24 May 2019 11:17:06 -0400 Subject: [PATCH 2/4] ODE-1264 Remove microseconds to allow second-level tolerance for timestamp comparisons --- odevalidator/sequential.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/odevalidator/sequential.py b/odevalidator/sequential.py index 59886a1..f01dc08 100644 --- a/odevalidator/sequential.py +++ b/odevalidator/sequential.py @@ -29,15 +29,15 @@ def validate_bundle(self, sorted_bundle): first_record = sorted_bundle[0] old_record_id = int(first_record['metadata']['serialId']['recordId']) old_serial_number = int(first_record['metadata']['serialId']['serialNumber']) - old_record_generated_at = dateutil.parser.parse(first_record['metadata']['recordGeneratedAt']) - old_ode_received_at = dateutil.parser.parse(first_record['metadata']['odeReceivedAt']) + old_record_generated_at = dateutil.parser.parse(first_record['metadata']['recordGeneratedAt']).replace(microsecond=0) + old_ode_received_at = dateutil.parser.parse(first_record['metadata']['odeReceivedAt']).replace(microsecond=0) validation_results = [] for record in sorted_bundle[1:]: new_record_id = int(record['metadata']['serialId']['recordId']) new_serial_number = int(record['metadata']['serialId']['serialNumber']) - new_record_generated_at = dateutil.parser.parse(record['metadata']['recordGeneratedAt']) - new_ode_received_at = dateutil.parser.parse(record['metadata']['odeReceivedAt']) + new_record_generated_at = dateutil.parser.parse(record['metadata']['recordGeneratedAt']).replace(microsecond=0) + new_ode_received_at = dateutil.parser.parse(record['metadata']['odeReceivedAt']).replace(microsecond=0) if 'metadata.serialId.recordId' not in self.skip_validations and record['metadata']['serialId']['bundleSize'] > 1 and new_record_id != old_record_id+1: validation_results.append(FieldValidationResult(False, "Detected incorrectly incremented recordId. Expected recordId '%d' but got '%d'" % (old_record_id+1, new_record_id), serial_id = record['metadata']['serialId'])) From 3d231eb8c4edaacffb74d8d31919ab2b26189a36 Mon Sep 17 00:00:00 2001 From: Matthew Schwartz Date: Fri, 24 May 2019 11:23:28 -0400 Subject: [PATCH 3/4] ODE-1264 Fix unit tests, replace assertEquals, comment print statements --- tests/__init__.py | 8 ++++---- tests/result_test.py | 16 ++++++++-------- tests/sequential_test.py | 13 ++++++------- tests/validator_field_test.py | 11 +++++------ 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 3e1ec2f..d8c6917 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -7,14 +7,14 @@ def assert_results(testcase, results, expected_fail_count): for val in res.field_validations: if not val.valid: if not result_partition_printed: - print("\n========") + # print("\n========") result_partition_printed = True serial_id = res.serial_id if not serial_id: serial_id = val.serial_id - - print("SerialId: %s, Field: %s, Details: %s\n--------\n%s" % (serial_id , val.field_path, val.details, res.record)) + + # print("SerialId: %s, Field: %s, Details: %s\n--------\n%s" % (serial_id , val.field_path, val.details, res.record)) fail_count += 1 - testcase.assertEquals(expected_fail_count, fail_count, "Expected %s failures, got %s failures." % (expected_fail_count, fail_count)) + testcase.assertEqual(expected_fail_count, fail_count, "Expected %s failures, got %s failures." % (expected_fail_count, fail_count)) diff --git a/tests/result_test.py b/tests/result_test.py index f738e73..6cfe152 100644 --- a/tests/result_test.py +++ b/tests/result_test.py @@ -7,14 +7,14 @@ class ResultTest(unittest.TestCase): def testFieldValidationResult(self): f = FieldValidationResult() self.assertTrue(f.valid) - self.assertEquals("", f.details) - self.assertEquals(None, f.field_path) - self.assertEquals('{"Field": null, "Valid": true, "Details": "", "SerialId": null}', json.dumps(f.to_json())) - self.assertEquals('{"Field": null, "Valid": true, "Details": "", "SerialId": null}', str(f)) + self.assertEqual("", f.details) + self.assertEqual(None, f.field_path) + self.assertEqual('{"Field": null, "Valid": true, "Details": "", "SerialId": null}', json.dumps(f.to_json())) + self.assertEqual('{"Field": null, "Valid": true, "Details": "", "SerialId": null}', str(f)) def testRecordValidationResult(self): f = RecordValidationResult("serial_id", [FieldValidationResult()], "record") - self.assertEquals("serial_id", f.serial_id) - self.assertEquals("record", f.record) - self.assertEquals('{"SerialId": "serial_id", "Validations": [{"Field": null, "Valid": true, "Details": "", "SerialId": null}], "Record": "record"}', json.dumps(f.to_json())) - self.assertEquals('{"SerialId": "serial_id", "Validations": [{"Field": null, "Valid": true, "Details": "", "SerialId": null}], "Record": "record"}', str(f)) + self.assertEqual("serial_id", f.serial_id) + self.assertEqual("record", f.record) + self.assertEqual('{"SerialId": "serial_id", "Validations": [{"Field": null, "Valid": true, "Details": "", "SerialId": null}], "Record": "record"}', json.dumps(f.to_json())) + self.assertEqual('{"SerialId": "serial_id", "Validations": [{"Field": null, "Valid": true, "Details": "", "SerialId": null}], "Record": "record"}', str(f)) diff --git a/tests/sequential_test.py b/tests/sequential_test.py index bfd0d4d..c613940 100644 --- a/tests/sequential_test.py +++ b/tests/sequential_test.py @@ -17,13 +17,13 @@ def setUp(self): self.record_list = self.build_happy_path(self.json_seed) def test_happy_path(self): - print("Testing Happy Path ...") + # print("Testing Happy Path ...") record_list = self.build_happy_path(self.json_seed) results = self.seq.perform_sequential_validations(record_list) assert_results(self, results, 0) def test_missing_records(self): - print("Testing Missing recordId, serialNumber ...") + # print("Testing Missing recordId, serialNumber ...") self.record_list.remove(self.record_list[19]) self.record_list.remove(self.record_list[8]) self.record_list.remove(self.record_list[2]) @@ -31,16 +31,16 @@ def test_missing_records(self): assert_results(self, results, 7) def test_invalid_bundle_size(self): - print("Testing invalid bundleSize ...") + # print("Testing invalid bundleSize ...") self.record_list.remove(self.record_list[15]) self.record_list.remove(self.record_list[6]) results = self.seq.perform_sequential_validations(self.record_list) - # Even though we have removed the last record of a full bundle, the validator can't detect if this is a head of a full list or a full list. + # Even though we have removed the last record of a full bundle, the validator can't detect if this is a head of a full list or a full list. # Therefore, we should get only one validation error assert_results(self, results, 1) def test_dup_and_chronological(self): - print("Testing Duplicate recordId and serialNumber and non-chronological odeReceivedAt and recordGeneratedAt ...") + # print("Testing Duplicate recordId and serialNumber and non-chronological odeReceivedAt and recordGeneratedAt ...") self.record_list[18] = copy.deepcopy(self.record_list[16]) self.record_list[9] = copy.deepcopy(self.record_list[7]) self.record_list[2] = copy.deepcopy(self.record_list[0]) @@ -89,7 +89,7 @@ def _build_bundle(self, seed_record, count): bundle.append(cur_record) cur_record = self._next_record(cur_record) count -= 1 - + return bundle def _next_record(self, cur_record): @@ -101,4 +101,3 @@ def _next_record(self, cur_record): next_record['metadata']['odeReceivedAt'] = received_at.isoformat() next_record['metadata']['recordGeneratedAt'] = generated_at.isoformat() return next_record - diff --git a/tests/validator_field_test.py b/tests/validator_field_test.py index df02c74..55209ca 100644 --- a/tests/validator_field_test.py +++ b/tests/validator_field_test.py @@ -21,7 +21,7 @@ def test_constructor_fails_no_type(self): self.assertEqual("Missing required configuration property 'Type' for field a.b.c={}", str(e)) def test_constructor_fails_no_field_config(self): - self.assertEquals('{"Path": "a.b.c", "Type": null, "UpperLimit": null, "LowerLimit": null, "Values": null, "EqualsValue": null, "EarliestTime": null, "LatestTime": null, "AllowEmpty": null}', str(Field("a.b.c"))) + self.assertEqual('{"Path": "a.b.c", "Type": null, "UpperLimit": null, "LowerLimit": null, "Values": null, "EqualsValue": null, "EarliestTime": null, "LatestTime": null, "AllowEmpty": null}', str(Field("a.b.c"))) def test_constructor_fails_invalid_earliesttime(self): input_field_object = {"Type":"timestamp", "EarliestTime":"invalidtimestamp"} @@ -34,7 +34,7 @@ def test_constructor_fails_invalid_earliesttime(self): def test_constructor_succeeds_valid_earliesttime(self): input_field_object = {"Type":"timestamp", "EarliestTime":"2019-03-14T14:54:21.596Z"} actual_field = Field("a.b.c", input_field_object) - self.assertEqual("2019-03-14 14:54:21.596000+00:00", str(actual_field.earliest_time)) + self.assertEqual("2019-03-14 14:54:21+00:00", str(actual_field.earliest_time)) def test_constructor_fails_invalid_latesttime(self): input_field_object = {"Type":"timestamp", "LatestTime":"invalidtimestamp"} @@ -47,7 +47,7 @@ def test_constructor_fails_invalid_latesttime(self): def test_constructor_succeeds_valid_latesttime(self): input_field_object = {"Type":"timestamp", "LatestTime":"2019-03-14T14:54:21.596Z"} actual_field = Field("a.b.c", input_field_object) - self.assertEqual("2019-03-14 14:54:21.596000+00:00", str(actual_field.latest_time)) + self.assertEqual("2019-03-14 14:54:21+00:00", str(actual_field.latest_time)) def test_constructor_succeeds_latesttime_now_keyword(self): input_field_object = {"Type":"timestamp", "LatestTime":"NOW"} @@ -169,6 +169,5 @@ def test_validate_returns_success_timestamp_after_earliest_time(self): def testFieldSerialization(self): test_field_object = {"Type":"timestamp", "EarliestTime":"2019-03-14T14:54:20.000Z"} test_field = Field("a.b.c", test_field_object) - self.assertEquals('{"Path": "a.b.c", "Type": "timestamp", "UpperLimit": null, "LowerLimit": null, "Values": null, "EqualsValue": null, "EarliestTime": "2019-03-14T14:54:20+00:00", "LatestTime": null, "AllowEmpty": false}', json.dumps(test_field.to_json())) - self.assertEquals('{"Path": "a.b.c", "Type": "timestamp", "UpperLimit": null, "LowerLimit": null, "Values": null, "EqualsValue": null, "EarliestTime": "2019-03-14T14:54:20+00:00", "LatestTime": null, "AllowEmpty": false}', str(test_field)) - + self.assertEqual('{"Path": "a.b.c", "Type": "timestamp", "UpperLimit": null, "LowerLimit": null, "Values": null, "EqualsValue": null, "EarliestTime": "2019-03-14T14:54:20+00:00", "LatestTime": null, "AllowEmpty": false}', json.dumps(test_field.to_json())) + self.assertEqual('{"Path": "a.b.c", "Type": "timestamp", "UpperLimit": null, "LowerLimit": null, "Values": null, "EqualsValue": null, "EarliestTime": "2019-03-14T14:54:20+00:00", "LatestTime": null, "AllowEmpty": false}', str(test_field)) From ccb1d8a7d6db825788818f1cb14e156602ca37c3 Mon Sep 17 00:00:00 2001 From: Matthew Schwartz Date: Fri, 24 May 2019 11:27:42 -0400 Subject: [PATCH 4/4] ODE-1264 Update docs --- README.md | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index fed2b18..86a0937 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,7 @@ class RecordValidationResult: ``` **Response Syntax** -The following is a truncated examples of the `validate_queue` response. Full sample can be +The following is a truncated examples of the `validate_queue` response. Full sample can be viewed by following the provided link: ``` @@ -194,7 +194,7 @@ viewed by following the provided link: "Field": "metadata.recordGeneratedBy", "Valid": true, "Details": "" - }, + }, ... ], "Record": { @@ -312,7 +312,7 @@ properties: type: array items: "$ref": "#/definitions/Conditions" - + definitions: Conditions: ifPart: @@ -353,14 +353,14 @@ definitions: items: type: "$ref": "#/definitions/ThenPart" - + ThenPart: description: This is the data schema for `thenPart` component of `EqualsValue` config option type: object properties: startsWithField: description: The value of this property is a fully qualified path to another data field. - The value of this field is expected to `start` with the value of + The value of this field is expected to `start` with the value of the referenced data field. type: string @@ -370,7 +370,7 @@ definitions: type: array items: type: string - + skipSequentialValidation: description: This boolean property specifies if this sequential or chronological field should take part in sequential validation or not. set the value of this property to True if you would like to skip sequential validation of this field. @@ -484,16 +484,16 @@ EqualsValue = {"startsWithField": "metadata.recordType"} - Summary: Used with decimal types to specify the highest acceptable value for this field - Value: decimal number: `UpperLimit = 150.43` - `EarliestTime` \[_optional_\] - - Summary: Used with timestamp types to specify the earliest acceptable timestamp for this field + - Summary: Used with timestamp types to specify the earliest acceptable timestamp for this field down to second-level precision - Value: ISO timestamp: `EarliestTime = 2018-12-03T00:00:00.000Z` - Note: For more information on how to write parsable timestamps, see [dateutil.parser.parse()](https://dateutil.readthedocs.io/en/stable/parser.html#dateutil.parser.parse). - `LatestTime` \[_optional_\] - - Summary: Used with timestamp types to specify the latest acceptable timestamp for this field + - Summary: Used with timestamp types to specify the latest acceptable timestamp for this field down to second-level precision - Special value: Use `NOW` to validate that the timestamp is not in the future: `LatestTime = NOW` - Value: ISO timestamp: `LatestTime = 2018-12-03T00:00:00.000Z` - Note: For more information on how to write parsable timestamps, see [dateutil.parser.parse()](https://dateutil.readthedocs.io/en/stable/parser.html#dateutil.parser.parse). -The following field validation specifies that sequential validation should NOT be enacted on `metadata.recordGeneratedAt` when the record is generated +The following field validation specifies that sequential validation should NOT be enacted on `metadata.recordGeneratedAt` when the record is generated by TMC (`"metadata.recordGeneratedBy":"TMC"`). ``` @@ -505,7 +505,7 @@ EqualsValue = {"conditions":[{"ifPart":{"fieldName":"metadata.recordGeneratedBy" The following field validation specifies that sequential validation should NOT be enacted on `metadata.serialId.recordId` when the records is from and `rxMsg` OR the records is _santiized_ (`"metadata.sanitized": "True"`. -fields when +fields when ``` [metadata.serialId.recordId] @@ -573,6 +573,9 @@ This library is used in the following test and verification applications as of t ## Release Notes +### Release 0.0.6 +- Reduced precision of timestamp parsing to second-level instead of microsecond-level to allow roughly 1 second of tolerance + ### Release 0.0.5 - Added support for CSV files - Added `--config-file` command-line argument @@ -592,15 +595,15 @@ is always a possibility that two keys at different structures be named the same. has to be the ful path of the field name. * Took advantage of python configuration file variable dereferencing * `config.ini` `then_part` can now be an empty array or completely missing which would mean that the field is -optional if the `if_part` condition is met. +optional if the `if_part` condition is met. * The `fieldValue` of the `ifPart` can also be eliminated which would mean that the if the field identified by -the `fieldName` of the `ifPart` exists, the field has to be validated. If it the field doesn't exist, +the `fieldName` of the `ifPart` exists, the field has to be validated. If it the field doesn't exist, it is considered optional and no validation error is issued. * Fields can now be specified to allow empty value by setting `AllowEmpty = True`. Currently only -`elevation` is allowed to be empty. +`elevation` is allowed to be empty. * Added field info to `FieldValidationResult` (renamed from ValidationResult) objects * Added serialId to the `RecordValidationResult` (NEW) object. This replaces RecordID. -* `Field, FieldValidationResult and RecordValidationResult` objects now have `__str__()` method +* `Field, FieldValidationResult and RecordValidationResult` objects now have `__str__()` method which allows them to be serialized for printing purposes. * Test files `good.json` and `bad.json` were updated with new tests for Broadcast TIM