fix: Modified to not run the full metadata tests in some circumstances (

#673)

The health check does only basic metadata checking now if the trigger comes from the catalog repo

.github/workflows/integration-tests.yml

@@ -50,13 +50,23 @@ jobs:
         pip install -r requirements.txt
 
     - name: Health Check
-      run: ./scripts/integration-tests.sh -u ${{ inputs.API_URL }} -f $FILE_PATH -c MetadataEndpointTests
+      # When triggered by a repo dispatch the API code is not deployed so there could be discrepancies
+      # between the code running the tests and the code running the API. In that case don't do the
+      # MetadataEndpointTests since it makes exact comparisons of the hash and version.
+      # The BasicMetadataEndpointTests just looks at the hash and version format, without looking
+      # at the exact values, so always do these tests.
+      run: |
+        health_check_classes="BasicMetadataEndpointTests"
+        if [[ "${{ github.event_name }}" == "repository_dispatch" ]]; then
+          health_check_classes += ",MetadataEndpointTests"
+        fi
+        ./scripts/integration-tests.sh -u ${{ inputs.API_URL }} -f $FILE_PATH -c $health_check_classes
       env:
         FILE_PATH: ${{ env.FILE_PATH }}
         REFRESH_TOKEN: ${{ secrets.REFRESH_TOKEN }}
 
     - name: Run Integration Tests
-      run: ./scripts/integration-tests.sh -u ${{ inputs.API_URL }} -f $FILE_PATH
+      run: ./scripts/integration-tests.sh -u ${{ inputs.API_URL }} -f $FILE_PATH -x MetadataEndpointTests
       env:
         FILE_PATH: ${{ env.FILE_PATH }}
         REFRESH_TOKEN: ${{ secrets.REFRESH_TOKEN }}
@@ -69,7 +79,3 @@ jobs:
         path: |
           integration-tests/src/integration_tests_log.html
           integration-tests/src/datasets_validation.csv
-          
-          
-          
-

integration-tests/src/endpoints/integration_tests.py

@@ -163,7 +163,7 @@ def get_test_methods_for_class(cls):
             and method_name != "test_all"
         ]
 
-    def test_all(self, target_classes=[]):
+    def test_all(self, target_classes=[], excluded_classes=[]):
         """Test all endpoints."""
         console = self.console
         console.clear()
@@ -198,7 +198,12 @@ def test_all(self, target_classes=[]):
         test_methods = [
             (child, self.get_test_methods_for_class(child))
             for child in child_classes
-            if child.__name__ in target_classes or not len(target_classes)
+            if (child.__name__ in target_classes if len(target_classes) > 0 else True)
+            and (
+                child.__name__ not in excluded_classes
+                if len(excluded_classes) > 0
+                else True
+            )
         ]
 
         n_tests = sum(len(methods) for _, methods in test_methods)

integration-tests/src/endpoints/metadata.py

@@ -4,58 +4,90 @@
 from endpoints.integration_tests import IntegrationTests
 
 
-class MetadataEndpointTests(IntegrationTests):
+class BasicMetadataEndpointTests(IntegrationTests):
+    """
+    This class is used to perform basic tests on the metadata endpoint of the API.
+    It will check if the format of the hash and version obtained from the API is correct.
+    """
+
     def __init__(self, file_path, access_token, url, progress):
         super().__init__(file_path, access_token, url, progress=progress)
-        self.version_info_path = os.path.join(
-            os.path.dirname(__file__), "../../../api/src/version_info"
-        )
-
-    def read_version_info(self):
-        """Read the version_info file and extract the long commit hash and extracted version"""
-        with open(self.version_info_path, "r") as file:
-            content = file.read()
-            long_commit_hash = re.search(r"LONG_COMMIT_HASH=(\w+)", content).group(1)
-            extracted_version = re.search(r"EXTRACTED_VERSION=(\S+)", content).group(1)
-        return long_commit_hash, extracted_version
 
-    def test_metadata(self):
+    @staticmethod
+    def read_version_info_from_api(integrationTestsObject):
         """Test retrieval of GTFS feeds"""
-        response = self.get_response("v1/metadata")
+        response = integrationTestsObject.get_response("v1/metadata")
         assert (
             response.status_code == 200
         ), f"Expected 200 status code for metadata, got {response.status_code}."
         metadata = response.json()
+        api_version = metadata["version"]
+        assert (
+            api_version is not None
+        ), f"Could not get proper version from API metadata = {metadata}"
 
-        version = metadata["version"]
+        api_commit_hash = metadata["commit_hash"]
         assert (
-            version is not None
-        ), f"Could not extract version from metadata = {metadata}"
+            api_commit_hash is not None
+        ), f"Could not get proper commit_hash from API metadata = {metadata}"
 
-        assert re.match(
-            r"^v\d+\.\d+\.\d+(_SNAPSHOT)?$", version
-        ), f"Found version but not the right format. metadata = {metadata}"
+        return api_commit_hash, api_version
 
-        commit_hash = metadata["commit_hash"]
-        assert (
-            commit_hash is not None
-        ), f"Could not extract commit_hash from metadata = {metadata}"
+    def test_metadata(self):
+        (
+            api_commit_hash,
+            api_version,
+        ) = BasicMetadataEndpointTests.read_version_info_from_api(self)
+        assert re.match(
+            r"^v\d+\.\d+\.\d+(_SNAPSHOT)?$", api_version
+        ), f"version from the API not the right format. version = {api_version}"
 
         # For the commit hash, it's safe to say it should be more than 20 characters.
         assert (
-            len(commit_hash) > 20
-        ), f"Commit hash seems too short in metadata = {metadata}"
+            len(api_commit_hash) > 20
+        ), f"Commit hash from API seems too short  = {api_commit_hash}"
+
+
+class MetadataEndpointTests(IntegrationTests):
+    """
+    This class is used to perform more detailed tests on the metadata endpoint of the API.
+    It check if the hash and version from the API are the proper ones.
+    Don't call this if the API and tests run on different versions.
+    """
+
+    def __init__(self, file_path, access_token, url, progress):
+        super().__init__(file_path, access_token, url, progress=progress)
+        self.version_info_path = os.path.join(
+            os.path.dirname(__file__), "../../../api/src/version_info"
+        )
+
+    def read_version_info_from_file(self):
+        """Read the version_info file and extract the long commit hash and extracted version"""
+        with open(self.version_info_path, "r") as file:
+            content = file.read()
+            long_commit_hash = re.search(r"LONG_COMMIT_HASH=(\w+)", content).group(1)
+            extracted_version = re.search(r"EXTRACTED_VERSION=(\S+)", content).group(1)
+        return long_commit_hash, extracted_version
+
+    def test_metadata(self):
+        (
+            api_commit_hash,
+            api_version,
+        ) = BasicMetadataEndpointTests.read_version_info_from_api(self)
 
         # Read the expected values from the version_info file
-        expected_long_commit_hash, expected_extracted_version = self.read_version_info()
+        (
+            expected_long_commit_hash,
+            expected_extracted_version,
+        ) = self.read_version_info_from_file()
 
         # Verify that the commit hash matches the long commit hash from the version_info file
-        assert commit_hash == expected_long_commit_hash, (
-            f"Commit hash from metadata ({commit_hash}) does not match expected long commit hash "
+        assert api_commit_hash == expected_long_commit_hash, (
+            f"Commit hash from metadata ({api_commit_hash}) does not match expected long commit hash "
             f"({expected_long_commit_hash})"
         )
 
         # Verify that the version matches the extracted version from the version_info file
         assert (
-            version == expected_extracted_version
-        ), f"Version from metadata ({version}) does not match expected extracted version ({expected_extracted_version})"
+            api_version == expected_extracted_version
+        ), f"Version from api ({api_version}) does not match expected extracted version ({expected_extracted_version})"

integration-tests/src/main.py

@@ -43,11 +43,23 @@ def set_up_configs():
         default="",
         type=parse_class_list,
     )
+    parser.add_argument(
+        "--exclude_classes",
+        help="Optional, comma-separated list of test class names to exclude.",
+        default="",
+        type=parse_class_list,
+    )
     args = parser.parse_args()
     access_token = os.getenv("ACCESS_TOKEN", None)
     if access_token is None:
         raise ValueError("ACCESS_TOKEN environment variable is not set.")
-    return args.file_path, access_token, args.url, args.include_classes
+    return (
+        args.file_path,
+        access_token,
+        args.url,
+        args.include_classes,
+        args.exclude_classes,
+    )
 
 
 if __name__ == "__main__":
@@ -60,8 +72,14 @@ def set_up_configs():
         if not ispkg:
             importlib.import_module(modname)
 
-    data_file_path, api_access_token, api_url, include_classes = set_up_configs()
+    (
+        data_file_path,
+        api_access_token,
+        api_url,
+        include_classes,
+        exclude_classes,
+    ) = set_up_configs()
 
     IntegrationTests(data_file_path, api_access_token, api_url).test_all(
-        target_classes=include_classes
+        target_classes=include_classes, excluded_classes=exclude_classes
     )

scripts/integration-tests.sh

@@ -8,25 +8,28 @@
 #   -u  URL of the API to test against
 #   -f  File path for the data file to be used in tests
 #   -c  Optional, comma-separated list of test class names to include
+#   -x  Optional, comma-separated list of test class names to exclude
 
 function print_help() {
-  echo "Usage: $0 -u <API URL> -f <FILE PATH> [-c <CLASS NAMES>]"
+  echo "Usage: $0 -u <API URL> -f <FILE PATH> [-c <CLASS NAMES>] [-x <CLASS NAMES>]"
   echo ""
   echo "Options:"
   echo "  -u  URL of the API to test against"
   echo "  -f  File path for the data file to be used in tests"
   echo "  -c  Optional, comma-separated list of test class names to include"
+  echo "  -x  Optional, comma-separated list of test class names to exclude."
   exit 1
 }
 
 # Initialize include_classes as an empty string
 INCLUDE_CLASSES=""
 
-while getopts ":u:f:c:h" opt; do
+while getopts ":u:f:c:x:h" opt; do
   case ${opt} in
     u ) URL=$OPTARG ;;
     f ) FILE_PATH=$OPTARG ;;
     c ) INCLUDE_CLASSES=$OPTARG ;;
+    x ) EXCLUDE_CLASSES=$OPTARG ;;
     h ) print_help ;;
     \? ) print_help ;;
   esac
@@ -80,9 +83,13 @@ pip install -r "$PARENT_DIR"/integration-tests/requirements.txt &> /dev/null
 
 export PYTHONPATH="${PARENT_DIR}:${PARENT_DIR}/integration-tests/src:${PARENT_DIR}/api/src"
 
-# Pass include_classes if it's not empty
-if [ -z "${INCLUDE_CLASSES}" ]; then
-  (cd "$PARENT_DIR"/integration-tests/src && python "$PARENT_DIR"/integration-tests/src/main.py --file_path "${FILE_PATH}" --url "$URL")
-else
-  (cd "$PARENT_DIR"/integration-tests/src && python "$PARENT_DIR"/integration-tests/src/main.py --file_path "${FILE_PATH}" --url "$URL" --include_classes "${INCLUDE_CLASSES}")
+classes_args=""
+if [ -n "${INCLUDE_CLASSES}" ]; then
+  classes_args="--include_classes ${INCLUDE_CLASSES}"
+fi
+
+if [ -n "${EXCLUDE_CLASSES}" ]; then
+  classes_args+=" --exclude_classes ${EXCLUDE_CLASSES}"
 fi
+
+(cd "$PARENT_DIR"/integration-tests/src && python "$PARENT_DIR"/integration-tests/src/main.py --file_path "${FILE_PATH}" --url "$URL" $classes_args)