diff --git a/extract/s3_extract/push_proxy_job.py b/extract/s3_extract/push_proxy_job.py deleted file mode 100755 index 8847214ef..000000000 --- a/extract/s3_extract/push_proxy_job.py +++ /dev/null @@ -1,12 +0,0 @@ -import argparse - -from extract.s3_extract.stage_import import push_proxy_import - -parser = argparse.ArgumentParser() -parser.add_argument("log_type", help="Type to import, can be US, TEST, or DE") -parser.add_argument("date", help="Date to execute import for") - -if __name__ == "__main__": - args = parser.parse_args() - - push_proxy_import(args.log_type, args.date) diff --git a/extract/s3_extract/stage_import.py b/extract/s3_extract/stage_import.py index ce9df5ad6..25764a0a2 100644 --- a/extract/s3_extract/stage_import.py +++ b/extract/s3_extract/stage_import.py @@ -54,33 +54,6 @@ def diagnostics_import(import_date): ) -def push_proxy_import(log_type, import_date): - """ - Function to load data from a previously set up Snowflake stage - for push proxy data from AWS ELB - - @log_type: Str with valid values of US, TEST, or DE - @import_date: Date with format "%Y/%m/%d" - """ - loc = PUSH_PROXY_LOCATIONS[log_type] - aws_account_id = os.getenv('AWS_ACCOUNT_ID') - az = loc['az'] - - extract_from_stage( - loc['table'], - loc['stage'], - 'push_proxy', - get_path(aws_account_id, az), - get_push_proxy_pattern(import_date), - os.environ.copy(), - ) - - -def get_push_proxy_pattern(import_date): - date = import_date.replace('/', '\\/') - return f".*{date}\\/.*" - - def get_diagnostics_pattern(loc, import_date): return f".*{loc}.{import_date}.*" diff --git a/tests/extract/s3_extract/test_stage_import.py b/tests/extract/s3_extract/test_stage_import.py index 4e27e967b..8fe8a90ba 100644 --- a/tests/extract/s3_extract/test_stage_import.py +++ b/tests/extract/s3_extract/test_stage_import.py @@ -6,9 +6,7 @@ extract_from_stage, get_diagnostics_pattern, get_path, - get_push_proxy_pattern, licenses_import, - push_proxy_import, releases_import, ) @@ -86,44 +84,6 @@ def test_get_path(aws_account_id, az, expected): assert get_path(aws_account_id, az) == expected -@pytest.mark.parametrize( - "import_date,expected", - [ - ("2022-10-01", ".*2022-10-01\\/.*"), - ("2022/10/01", ".*2022\\/10\\/01\\/.*"), - ], -) -def test_get_push_proxy_pattern(import_date, expected): - assert get_push_proxy_pattern(import_date) == expected - - -@pytest.mark.parametrize( - "location,table,stage,zone", - [ - ("US", "logs", "push_proxy_stage", "us-east-1"), - ("DE", "de_logs", "push_proxy_de_stage", "eu-central-1"), - ("TEST", "test_logs", "push_proxy_test_stage", "us-east-1"), - ], -) -def test_push_proxy_import(mocker, mock_environment, location, table, stage, zone): - # GIVEN: environment configured for handling push proxy import - # GIVEN: calls to extract from stage are captured - mock_extract = mocker.patch("extract.s3_extract.stage_import.extract_from_stage") - - # WHEN: push proxy job is triggered for a specific location and date - push_proxy_import(location, "2022/10/01") - - # THEN: expect extract to have been called once - mock_extract.assert_called_once_with( - table, - stage, - "push_proxy", - f"AWSLogs/test-aws-account-id/elasticloadbalancing/{zone}", - ".*2022\\/10\\/01\\/.*", - mock_environment, - ) - - def test_licenses_import(mocker, mock_environment): # GIVEN: environment configured for handling two diagnostic imports -- see mock_environment # GIVEN: calls to extract from stage are captured