diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..3914f04 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max_line_length = 88 diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 0000000..c65a1cc --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,46 @@ +--- +name: Docker Registry Scanner Main CI/CD Pipelines + +on: + push: + branches: [main] + paths-ignore: [README.md] + pull_request: + branches: [main] + paths-ignore: [README.md] + +defaults: + run: + shell: bash + +permissions: read-all + +jobs: + syntax: + name: Syntax + uses: ./.github/workflows/syntax.yaml + + tests: + name: Tests + needs: [syntax] + uses: ./.github/workflows/tests.yaml + with: + coverage_min: '95' + + tag_release: + name: Create Tag & Release + needs: [tests] + permissions: + contents: write + uses: ./.github/workflows/tag_release.yaml + with: + dry_run: |- + ${{ + github.event_name == 'pull_request' || + ( + github.event_name == 'push' && + github.ref != 'refs/heads/main' + ) + }} + secrets: + gh_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/syntax.yaml b/.github/workflows/syntax.yaml new file mode 100644 index 0000000..244fb44 --- /dev/null +++ b/.github/workflows/syntax.yaml @@ -0,0 +1,36 @@ +--- +name: Docker Registry Scanner Syntax Checks + +on: + workflow_call: + +permissions: read-all + +jobs: + syntax: + name: Syntax + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Commitlint + uses: wagoid/commitlint-github-action@v5 + with: + failOnWarnings: true + + - name: Install YAMLlint, PyLint, Flake8 and Black + run: python -m pip install --user pylint flake8 black yamllint + + - name: Run YAMLlint + run: yamllint --strict . + + - name: Run PyLint + run: python -m pylint --recursive yes . + + - name: Run Flake8 + run: python -m flake8 . + + - name: Run Black + run: python -m black --diff --check --color . diff --git a/.github/workflows/tag_release.yaml b/.github/workflows/tag_release.yaml new file mode 100644 index 0000000..43f4c76 --- /dev/null +++ b/.github/workflows/tag_release.yaml @@ -0,0 +1,51 @@ +--- +name: Docker Registry Scanner Release And Tag + +on: + workflow_call: + inputs: + dry_run: + description: Dry-run for creating tag and release + required: true + type: boolean + secrets: + gh_token: + description: GitHub token + required: true + outputs: + new_version: + description: New tag version generated using conventional commit + value: ${{ jobs.tag.outputs.new_version }} + +permissions: + contents: write + +jobs: + tag: + name: Tag + runs-on: ubuntu-latest + outputs: + changelog: ${{ steps.tag_version.outputs.changelog }} + new_tag: ${{ steps.tag_version.outputs.new_tag }} + new_version: ${{ steps.tag_version.outputs.new_version }} + steps: + - name: Bump Version & Tag + id: tag_version + uses: mathieudutour/github-tag-action@v6.1 + with: + default_bump: false + dry_run: ${{ inputs.dry_run }} + github_token: ${{ secrets.gh_token }} + + release: + if: ${{ !inputs.dry_run && needs.tag.outputs.new_tag != '' }} + name: Release + runs-on: ubuntu-latest + needs: [tag] + steps: + - name: GitHub Release + uses: ncipollo/release-action@v1 + with: + tag: ${{ needs.tag.outputs.new_tag }} + name: Release ${{ needs.tag.outputs.new_tag }} + body: ${{ needs.tag.outputs.changelog }} diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 0000000..ddb6d2c --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,49 @@ +--- +name: Docker Registry Scanner Tests + +on: + workflow_call: + inputs: + coverage_min: + description: Minimum code coverage + required: true + type: string + +permissions: read-all + +jobs: + tests: + name: Tests + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install Pytest And Coverage + run: python -m pip install pytest coverage + + - name: Run Unit Testing + env: + PYTHONPATH: src/ + run: python -m coverage run -m pytest -v tests/ + + - name: Check Coverage + id: coverage + continue-on-error: true + run: coverage report -m --fail-under ${{ inputs.coverage_min }} --omit 'tests/*' + + - name: Coverage Result + run: | + if [ "${{ steps.coverage.outcome }}" == "failure" ] + then + echo "::warning::BNPPF code coverage NOK 💀 (<${{ inputs.coverage_min }}%)" + exit 1 + else + echo "::notice::BNPPF code coverage OK 🎉 (>${{ inputs.coverage_min }}%)" + fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0262971 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +certs/ +!tests/certs/ +.coverage +__pycache__ +.pytest_cache +scan_results_report.json diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..3d310e8 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,2 @@ +[pylint] +disable = import-error diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000..ef80e3b --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,7 @@ +--- +extends: default +rules: + line-length: + max: 100 + truthy: + check-keys: false diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..80ba2ad --- /dev/null +++ b/Dockerfile @@ -0,0 +1,46 @@ +ARG PYTHON_VERSION="3.11" + +FROM debian:12-slim AS gpg-dearmor + +COPY pubkeys/trivy.key /tmp + +# DL3008 warning: Pin versions in apt get install +# hadolint ignore=DL3008 +RUN apt-get update \ + && apt-get install --no-install-recommends -y gnupg \ + && apt-get upgrade -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && gpg --dearmor > /tmp/trivy.gpg < /tmp/trivy.key + +FROM python:"${PYTHON_VERSION}-slim" + +LABEL org.opencontainers.image.authors="Jonathan Sabbe " + +WORKDIR /app + +COPY src/ . +COPY apt/trivy.sources /etc/apt/sources.list.d/ +COPY --from=gpg-dearmor /tmp/trivy.gpg /etc/apt/trusted.gpg.d/trivy.gpg + +ARG TRIVY_VERSION="0.45.1" + +# DL3008 warning: Pin versions in apt get install +# hadolint ignore=DL3008 +RUN apt-get update \ + && apt-get install --no-install-recommends -y git trivy=$TRIVY_VERSION \ + && apt-get upgrade -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && adduser --disabled-password --comment "Docker Scanner" scanner \ + && chown -R scanner:scanner . \ + && pip uninstall -y setuptools + +USER scanner + +RUN git config --global --add safe.directory "${PWD}" + +ENV TZ="Europe/Brussels" \ + PYTHONUNBUFFERED="1" + +ENTRYPOINT [ "python", "main.py" ] diff --git a/README.md b/README.md new file mode 100644 index 0000000..18966da --- /dev/null +++ b/README.md @@ -0,0 +1,76 @@ +# Docker Registry Scanner + +## Description + +The goal of this application is to scan Docker images stored on a [registry](https://github.com/docker) (for now only registry made by [Docker Inc.](https://github.com/docker)). + +Scan is done using Trivy security scanner. + +Docker Registry Scanner application writes status of images as standard output and scan results report JSON file (defaults: `./scan_results_report.json`) is created at the end of the run. + +**Example** + +```shell +Scanning Docker image 'localhost:443/my-poc:latest'... +🔴 NOK localhost:443/my-poc:latest ({'HIGH': 1, 'CRITICAL': 0}) +Scanning Docker image 'localhost:443/semver-bumper:08dc6233'... +Scanning Docker image 'localhost:443/semver-bumper:latest'... +🔴 NOK localhost:443/semver-bumper:08dc6233 ({'HIGH': 1, 'CRITICAL': 0}) +🟢 OK localhost:443/semver-bumper:latest +Scanning Docker image 'localhost:443/ubuntu:latest'... +🟢 OK localhost:443/ubuntu:latest +``` + +## Configuration + +### Environment Variables + +- `LOGGING_LEVEL`: (Optional) Logging level needed. Can be `DEBUG`, `INFO`, `WARNING` or `CRITICAL`. (Default: `INFO`) +- `DOCKER_REGISTRY_URL`: **(Required)** Docker registry **HTTPS** URL that needs to be scanned. (e.g. `https://docker-registry.example.com:12345/path/to/repository/`) +- `DOCKER_REGISTRY_CA_FILE`: (Optional) PEM format file of CA. +- `DOCKER_IMAGES_FILTER`: (Optional) REGEX pattern used to filter Docker images. (Default: `.*`) +- `DOCKER_TAGS_FILTER`: (Optional) REGEX pattern used to filter Docker image tags. (Default: `.*`) +- `IMAGE_LIST_NBR_MAX`: (Optional) Maximum number of Docker images that needs to be fetch from Docker registry. (Default: `1000`) +- `HTTPS_CONNECTION_TIMEOUT`: (Optional) Docker registry client HTTPS connection timeout. (Default: `3`) +- `SCAN_SEVERITY`: (DEPRECATED) Scanner severity configuration. Deprecated in favor of `SCAN_MIN_SEVERITY`. (Default: `HIGH,CRITICAL`) +- `SCAN_MIN_SEVERITY`: (Optional) Scanner minimum severity threshold. Can be `UNKNOWN`, `LOW`, `MEDIUM`, `HIGH` or `CRITICAL`. (Defaut: `HIGH`) +- `SCAN_RESULTS_REPORT_FILE`: (Optional) Scanner results report file. (Default: `./scan_results_report.json`) +- `SCAN_SCANNERS`: (Optional) Scanner scan types to do. (Default: `vuln,secret`) +- `MULTIPROCESSING_PROCESSES`: (Optional): Process in parallel used to scan Docker images. (Default: `5`) + + +## Docker + +### Build + +```shell +docker build -t docker-registry-scanner . +``` + +### Run + +**Examples** + +```shell +# Minimum required +docker run \ + --rm \ + -e DOCKER_REGISTRY_URL="https://docker-registry.example.com:12345" \ + docker-registry-scanner + +# Filter Docker images, minimum scan severity 'LOW' and logging level 'DEBUG' +docker run \ + --rm \ + -e DOCKER_REGISTRY_URL="https://docker-registry.example.com:12345" \ + -e DOCKER_IMAGES_FILTER='^release/docker/internal/speos/$' \ + -e SCAN_MIN_SEVERITY="LOW" \ + -e LOGGING_LEVEL="DEBUG" \ + docker-registry-scanner + +# HTTPS Proxy to reach Trivy databases URLs +docker run \ + --rm \ + -e DOCKER_REGISTRY_URL="https://docker-registry.example.com:12345" \ + -e https_proxy="http://proxy.example.com:7890" \ + docker-registry-scanner +``` diff --git a/apt/trivy.sources b/apt/trivy.sources new file mode 100644 index 0000000..d406eec --- /dev/null +++ b/apt/trivy.sources @@ -0,0 +1,5 @@ +Types: deb +URIs: https://aquasecurity.github.io/trivy-repo/deb +Suites: bookworm +Components: main +Signed-By: /etc/apt/trusted.gpg.d/trivy.gpg diff --git a/pubkeys/trivy.key b/pubkeys/trivy.key new file mode 100644 index 0000000..65de8dc --- /dev/null +++ b/pubkeys/trivy.key @@ -0,0 +1,23 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQGNBFzP66QBDAC+sYojv+2L2wU7ZOjekt741t/Hz9mAR6rZfsFE8eOdK6Ozav2t +l13QKpqSTUwQ33mQEwudEIUsZOov2tsBaG4r9Kxasu7wmeXoenvLWt92K6ZfcFwq +THti2Xcku6T1X+JXu4RAHf2wEreVY+Os8BnXMuIp4KWbFhVcLloeiMMZXUgSDHGM +pPXfvkF0D4AewSOlnlf8we7+AVX6ZA9tHyb2KVO40f3mgfwMA110B5xGRY5JU+hI +zouwwRb8xPX7cRf4BKaLKcMGzCEL+Ks7MUGCN7pKtIPkxDC1EyHk0dVOKKm2dl0W +kaZNTD+UoJjUTgkT/y+EJC2/nXMCNwdloGo+L9Ue8baRMku6PQEakmdh6j+y5Hwo +KafrJDK+Li+pnibN8TruEjIiJKbZWRWIiQ9Z3zY3O1KmrWilfFywICn9kyP9d1Rb +SCGx495cGzXl1RNB4k1j3oUOINL2A/5jIyh/e27/aNoqGad3xTMMdr2VmmxenTBz +xsJMXJViVD4CPacAEQEAAbQFdHJpdnmJAdQEEwEIAD4WIQQuLTVnRhYyyEu2zW/p +0KNhYnb6bAUCXM/rpAIbAwUJEswDAAULCQgHAgYVCAkKCwIEFgIDAQIeAQIXgAAK +CRDp0KNhYnb6bFBmDACtd1no/JcuBKDuntn0DCLw+j0A+v64GIXHJhyt2fxYhIG/ +WwTkkZ27glIbOW8EvsDoa13Ec7slPCTQI9WYn9hXJcMOXSnHyIWaVguIM2ouSYZK +H6N2g1lA+Fh3vjpat63Lysp1hpOYQckM37x6N2M/H/kfeLnMMo9e7tcjHsO07/2t +RJorOzMZR7zZmNR+bhh58caes7IRbhihp6dMEZBacQ8qcHlbFFEmyVt38nXreSgi +b4BuXgtOfFpZftUZX5eF7xhTYdIKOKKAn3pTpVdCP6UAseVtGwMUFUpfz6sXrPnc +Vvm6edFMOX+LkZ0SH6GT4XGqcFZNleuODL+kzTkqqmZ+M0tEeauprlBw364CgL5I +FVs2p2Wr7Gjn7TW7zOmzT8Cl6Lw5if5i+kH0ROh0JLfDhUa+bIrVKiXO2QF0tLl2 +p4GqyKKfp3BbHROF7bKqpypzB50tYpMMTCnIYQGyTtLTCv2SbO6PHepvXXKsODII +bfbBTsY/XQ0WYaD/HVs= +=HQ8K +-----END PGP PUBLIC KEY BLOCK----- diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..5e398ca --- /dev/null +++ b/src/config.py @@ -0,0 +1,23 @@ +"""Configuration""" + +from typing import Final, Optional +from os import getenv + +LOGGING_LEVEL: Final[str] = getenv(key="LOGGING_LEVEL", default="INFO") +DOCKER_REGISTRY_URL: Final[Optional[str]] = getenv(key="DOCKER_REGISTRY_URL") +DOCKER_REGISTRY_CA_FILE: Final[Optional[str]] = getenv(key="DOCKER_REGISTRY_CA_FILE") +DOCKER_IMAGES_FILTER: Final[str] = getenv(key="DOCKER_IMAGES_FILTER", default=r".*") +DOCKER_TAGS_FILTER: Final[str] = getenv(key="DOCKER_TAGS_FILTER", default=r".*") +IMAGE_LIST_NBR_MAX: Final[int] = int(getenv(key="IMAGE_LIST_NBR_MAX", default="1000")) +HTTPS_CONNECTION_TIMEOUT: Final[int] = int( + getenv(key="HTTP_CONNECTION_TIMEOUT", default="3") +) +SCAN_SEVERITY: Final[str] = getenv(key="SCAN_SEVERITY", default="HIGH,CRITICAL") +SCAN_MIN_SEVERITY: Final[Optional[str]] = getenv(key="SCAN_MIN_SEVERITY") +SCAN_RESULTS_REPORT_FILE: Final[str] = getenv( + key="SCAN_RESULTS_REPORT_FILE", default="./scan_results_report.json" +) +SCAN_SCANNERS: Final[str] = getenv(key="SCAN_SCANNERS", default="vuln,secret") +MULTIPROCESSING_PROCESSES: Final[int] = int( + getenv(key="MULTIPROCESSING_PROCESSES", default="5") +) diff --git a/src/docker_registry_client.py b/src/docker_registry_client.py new file mode 100644 index 0000000..2bd4363 --- /dev/null +++ b/src/docker_registry_client.py @@ -0,0 +1,77 @@ +"""Docker Registry Client""" + +from http.client import HTTPSConnection, HTTPResponse, HTTPException +from urllib.parse import urlparse, ParseResult +from json import loads +from re import search +from ssl import SSLContext, PROTOCOL_TLS_CLIENT, CERT_REQUIRED +from typing import Optional +from logger import logger + + +class DockerRegistryClient: + """DockerRegistryClient Class""" + + def __init__( + self, + registry_url: str, + ca_file: Optional[str] = None, + timeout: int = 3, + ) -> None: + if not registry_url.startswith("https://"): + raise ValueError("Docker registry URL must start with 'https://'") + + parse_result: ParseResult = urlparse(url=registry_url) + + self.registry_host: str = str(parse_result.hostname) + self.registry_port: int = parse_result.port if parse_result.port else 443 + self.registry_path = parse_result.path + + ssl_context: SSLContext = SSLContext( + protocol=PROTOCOL_TLS_CLIENT, verify_mode=CERT_REQUIRED + ) + if ca_file: + ssl_context.load_verify_locations(cafile=ca_file) + else: + ssl_context.load_default_certs() + self.https_connection = HTTPSConnection( # nosemgrep: bandit.B309 + host=self.registry_host, + port=self.registry_port, + timeout=timeout, + context=ssl_context, + ) + + def __request(self, url: str, method: str = "GET") -> bytes: + self.https_connection.request(method=method, url=url) + response: HTTPResponse = self.https_connection.getresponse() + if response.status != 200: + location_header: Optional[str] = response.getheader(name="location") + if location_header: + logger.info( + msg=f"💡 URL redirection detected: {url} -> {location_header}" + ) + _ = response.read() + return self.__request(url=location_header) + raise HTTPException( + f"Received HTTP code != 200: {response.status} -> {response.reason}" + ) + return loads(response.read()) + + def get_images(self, number_max: int = 500, pattern: str = r".*") -> list[str]: + """DockerClient Get Images Method""" + + images: list[str] = dict( + self.__request(url=f"{self.registry_path}/v2/_catalog?n={number_max}") + ).get("repositories") + + return [image for image in images if search(pattern=pattern, string=image)] + + def get_image_tags(self, image: str, pattern: str = r".*") -> list[str]: + """DockerClient Get Image Tags Method""" + + tags: list[str] = dict( + self.__request(url=f"{self.registry_path}/v2/{image}/tags/list") + ).get("tags") + if not tags: + return [] + return [f"{image}:{tag}" for tag in tags if search(pattern=pattern, string=tag)] diff --git a/src/logger.py b/src/logger.py new file mode 100644 index 0000000..41876fb --- /dev/null +++ b/src/logger.py @@ -0,0 +1,12 @@ +"""Logging Configuration""" + +from logging import getLogger, basicConfig +from config import LOGGING_LEVEL + +basicConfig( + format="%(asctime)s %(message)s", + encoding="utf-8", + datefmt="%Y-%m-%dT%H:%M:%S%z", # 1996-12-19T16:39:57-08:00 + level=LOGGING_LEVEL, +) +logger = getLogger(__name__) diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..8fb884f --- /dev/null +++ b/src/main.py @@ -0,0 +1,107 @@ +"""Main""" + +from typing import Any, Literal, Optional +from json import dumps +from docker_registry_client import DockerRegistryClient +from scanner import Scanner, download_database +from scanner_options import ScannerOptions +from logger import logger +import config + + +def display_results(results: dict[str, Any]) -> None: + """Display Scanner Results To Standard Output""" + + for image, result in dict(results).items(): + text: str = "" + + if result: + status: Optional[str] = dict(result).get("status") + status_emoji: Literal["🟢", "🔴"] = "🟢" if status == "OK" else "🔴" + text = f"{status_emoji} {status}\t{image}" + + if status != "OK": + summary: Optional[dict[str, int]] = dict( + dict(result).get("vulnerabilities") + ).get("summary") + text = f"{text} ({summary})" + else: + text = f"🔥 FAILED '{image}'" + logger.info(text) + + +def export_scan_results(scan_results: dict[str, Any], output_file: str) -> None: + """Export Scan Results Function""" + + if scan_results: + with open(file=output_file, mode="w", encoding="UTF-8") as file: + file.write(dumps(scan_results, indent=2)) + logger.info(msg=f"✨ Scan results exported on {output_file}") + + +def main(): # pragma: no cover + """Main Function""" + + if config.DOCKER_REGISTRY_URL is None: + raise ValueError( + "Docker registry needs to be defined using " + "'DOCKER_REGISTRY_URL' environment variable!" + ) + + total_tags_scanned: int = 0 + scan_results: dict[str, Any] = {} + client: DockerRegistryClient = DockerRegistryClient( + registry_url=config.DOCKER_REGISTRY_URL, + timeout=config.HTTPS_CONNECTION_TIMEOUT, + ca_file=config.DOCKER_REGISTRY_CA_FILE, + ) + images: list[str] = client.get_images( + number_max=config.IMAGE_LIST_NBR_MAX, pattern=config.DOCKER_IMAGES_FILTER + ) + scanner_options: ScannerOptions = ScannerOptions( + severity=config.SCAN_SEVERITY, + scanners=config.SCAN_SCANNERS, + processes=config.MULTIPROCESSING_PROCESSES, + min_severity=config.SCAN_MIN_SEVERITY, + ) + + logger.info(msg=f"💡 Number of Docker images: {len(images)}") + + download_database() + # download_database(database="java") + + for image in images: + image_tags: list[str] = client.get_image_tags( + image=image, pattern=config.DOCKER_TAGS_FILTER + ) + + if not image_tags: + logger.warning(msg=f"🤡 No Docker tags found for '{image}'") + continue + + logger.info(msg=f"💡 Number of Docker tags for '{image}': {len(image_tags)}") + + scanner: Scanner = Scanner( + docker_registry=f"{client.registry_host}:{client.registry_port}", + image_tags=image_tags, + options=scanner_options, + ) + + try: + for results in scanner.scan(): + scan_results.update(results) + display_results(results=results) + total_tags_scanned += len(image_tags) + logger.info(msg=f"💡 Total Docker tags scanned: {total_tags_scanned}") + except RuntimeError as exc: + logger.critical(exc) + export_scan_results( + scan_results=scan_results, output_file=config.SCAN_RESULTS_REPORT_FILE + ) + export_scan_results( + scan_results=scan_results, output_file=config.SCAN_RESULTS_REPORT_FILE + ) + + +if __name__ == "__main__": # pragma: no cover + main() diff --git a/src/scanner.py b/src/scanner.py new file mode 100644 index 0000000..e1bcba2 --- /dev/null +++ b/src/scanner.py @@ -0,0 +1,147 @@ +"""Scanner""" + +from subprocess import run, DEVNULL # nosemgrep: bandit.B404 +from typing import Any, Optional +from multiprocessing import Pool +from json import loads +from logger import logger +from scanner_options import ScannerOptions + + +def download_database(database: Optional[str] = None): + """Download Scanner Database Function""" + + db_str: str = "Vulnerability" + download_db_param: str = "--download-db-only" + + if database is not None and database.lower() == "java": + db_str = "Java Index" + download_db_param = "--download-java-db-only" + + logger.info(msg=f"Downloading Trivy {db_str} database...") + process = run( + args=["trivy", "image", download_db_param], + check=False, + stdout=DEVNULL, + stderr=DEVNULL, + ) + + if process.returncode != 0: + raise SystemExit( + f"Failed to download Trivy {db_str} database" + f" (exit code: {process.returncode})" + ) + + +class Scanner: + """Scanner Class""" + + docker_registry: str + image_tags: list[str] + severity: str + scanners: str + processes: int + + def __init__( + self, docker_registry: str, image_tags: list[str], options: ScannerOptions + ) -> None: + self.docker_registry = docker_registry + self.image_tags = image_tags + self.severity = options.severity + self.processes = options.processes + self.scanners = options.scanners + + def __parse_results( + self, results: list[dict[str, Any]] + ) -> dict[str, Any]: # pragma: no cover + """Trivy JSON file results parser private method""" + parsed_results: dict[str, list[str]] = {} + summary: dict[str, int] = {severity: 0 for severity in self.severity.split(",")} + + for result in results: + vulns: list[str] = [] + result_class: str = result.get("Class") + result_target: str = result.get("Target") + if result_class == "secret": + for vuln in result.get("Secrets"): + sev: str = dict(vuln).get("Severity") + rule_id: str = dict(vuln).get("RuleID") + vulns.append(f"{rule_id} ({sev})") + summary.update({sev: summary.get(sev) + 1}) + else: + if "Vulnerabilities" not in result: + continue + for vuln in result.get("Vulnerabilities"): + sev: str = dict(vuln).get("Severity") + vuln_id: str = dict(vuln).get("VulnerabilityID") + + if f"{vuln_id} ({sev})" not in vulns: + vulns.append(f"{vuln_id} ({sev})") + summary.update({sev: summary.get(sev) + 1}) + + parsed_results.update({f"{result_class} ({result_target})": sorted(vulns)}) + parsed_results.update({"summary": summary}) + + return parsed_results + + def run_scan(self, image_tag: str) -> dict[str, Any]: + """Run Scanner Method""" + status: str = "OK" + vulnerabilities: list[Any] = [] + docker_image_tag: str = f"{self.docker_registry}/{image_tag}" + process_args: list[str] = [ + "trivy", + "image", + "--ignore-unfixed", + "--insecure", + "--format", + "json", + "--severity", + self.severity, + "--exit-code", + "111", + "--scanners", + self.scanners, + docker_image_tag, + ] + + logger.debug(msg=f"💤 Scanning Docker image '{docker_image_tag}'...") + process = run(args=process_args, capture_output=True, check=False) + + print(process.returncode, process.stdout) + + if process.returncode not in [111, 0]: + return {docker_image_tag: {}} + + proc_stdout: dict[str, Any] = dict(loads(process.stdout)) + # .Results + image_results: list[dict[str, Any]] = proc_stdout.get("Results") + # .Metadata + image_metadata: dict[str, Any] = proc_stdout.get("Metadata") + # .Metadata.ImageID + image_id: str = image_metadata.get("ImageID") + # .Metadata.ImageConfig + image_config: dict[str, Any] = image_metadata.get("ImageConfig") + # .Metadata.ImageConfig.created + image_created: str = image_config.get("created") + # .Metadata.ImageConfig.config.Labels + image_labels: dict[str, str] = dict(image_config.get("config")).get("Labels") + + if process.returncode == 111: + status = "NOK" + vulnerabilities = self.__parse_results(results=image_results) + + return { + docker_image_tag: { + "status": status, + "created": image_created, + "id": image_id, + "labels": image_labels, + "vulnerabilities": vulnerabilities, + } + } + + def scan(self) -> list[dict[str, dict[str, Any]]]: # pragma: no cover + """Scan Method (Multiprocessing)""" + with Pool(processes=self.processes) as pool: + return pool.map(func=self.run_scan, iterable=self.image_tags) diff --git a/src/scanner_options.py b/src/scanner_options.py new file mode 100644 index 0000000..bade292 --- /dev/null +++ b/src/scanner_options.py @@ -0,0 +1,42 @@ +"""Scanner Options""" + +from typing import Optional + + +class ScannerOptions: + """Scanner Options Class""" + + severities: list[str] = ["UNKNOWN", "LOW", "MEDIUM", "HIGH", "CRITICAL"] + + def __init__( + self, + severity: str = "UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL", + min_severity: Optional[str] = None, + scanners: str = "vuln,secret", + processes: int = 4, + ) -> None: + self._severity = severity + self._min_severity = min_severity + self._scanners = scanners + self._processes = processes + + @property + def scanners(self) -> str: + """Get scanners""" + + return self._scanners + + @property + def processes(self) -> int: + """Get processes""" + + return self._processes + + @property + def severity(self) -> str: + """Get severity""" + + if self._min_severity is not None: + severity_index: int = self.severities.index(self._min_severity) + return ",".join(self.severities[severity_index:]) + return self._severity diff --git a/tests/certs/example.com.crt b/tests/certs/example.com.crt new file mode 100644 index 0000000..1514c99 --- /dev/null +++ b/tests/certs/example.com.crt @@ -0,0 +1,9 @@ +-----BEGIN CERTIFICATE----- +MIIBNzCB6qADAgECAhRSyqX07NsfPTIJJa7f7kXqcsE1FDAFBgMrZXAwFjEUMBIG +A1UEAwwLZXhhbXBsZS5jb20wHhcNMjMwOTE0MTAyNjEyWhcNMjQwOTEzMTAyNjEy +WjAWMRQwEgYDVQQDDAtleGFtcGxlLmNvbTAqMAUGAytlcAMhAMOsR8M7jObtZmqL +PFOxWd/ixagAQvXI2DhfbCaSY3pqo0owSDAnBgNVHREEIDAeggtleGFtcGxlLmNv +bYIJbG9jYWxob3N0hwR/AAABMB0GA1UdDgQWBBS8d/fVlfRZyNjJ2TVHCQcZkKR0 +XjAFBgMrZXADQQCeIFDDiPbQYMLbCeuljeHxDlulEjBmjHxyfYZFUpqFxmR/oagb +jQ67btuym4yCWBAjfDYT0dts4SvOgJbGFVEI +-----END CERTIFICATE----- diff --git a/tests/fixtures/trivy_scan_result.json b/tests/fixtures/trivy_scan_result.json new file mode 100644 index 0000000..3f11a0d --- /dev/null +++ b/tests/fixtures/trivy_scan_result.json @@ -0,0 +1,154 @@ +{ + "SchemaVersion": 2, + "ArtifactName": "docker-registry.example.com:12345/alpine:3.7", + "ArtifactType": "container_image", + "Metadata": { + "OS": { + "Family": "alpine", + "Name": "3.7.3", + "EOSL": true + }, + "ImageID": "sha256:6d1ef012b5674ad8a127ecfa9b5e6f5178d171b90ee462846974177fd9bdd39f", + "DiffIDs": [ + "sha256:3fc64803ca2de7279269048fe2b8b3c73d4536448c87c32375b2639ac168a48b" + ], + "RepoTags": [ + "alpine:3.7" + ], + "RepoDigests": [ + "alpine@sha256:8421d9a84432575381bfabd248f1eb56f3aa21d9d7cd2511583c68c9b7511d10" + ], + "ImageConfig": { + "architecture": "amd64", + "container": "953e5de88d11a4e81e21b1c7c1957519b8ff21e6e638981f21ea5ef845e308c4", + "created": "2019-03-07T22:19:53.447205048Z", + "docker_version": "18.06.1-ce", + "history": [ + { + "created": "2019-03-07T22:19:53.313789681Z", + "created_by": "/bin/sh -c #(nop) ADD file:aa17928040e31624cad9c7ed19ac277c5402c4b9ba39f834250affca40c4046e in / " + }, + { + "created": "2019-03-07T22:19:53.447205048Z", + "created_by": "/bin/sh -c #(nop) CMD [\"/bin/sh\"]", + "empty_layer": true + } + ], + "os": "linux", + "rootfs": { + "type": "layers", + "diff_ids": [ + "sha256:3fc64803ca2de7279269048fe2b8b3c73d4536448c87c32375b2639ac168a48b" + ] + }, + "config": { + "Cmd": [ + "/bin/sh" + ], + "Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ], + "Image": "sha256:534c86f7312dbbab3a8f724cc87fa82f0770ec171659112c975315a7a6166a94", + "ArgsEscaped": true + } + } + }, + "Results": [ + { + "Target": "docker-registry.example.com:12345/alpine:3.7 (alpine 3.7.3)", + "Class": "os-pkgs", + "Type": "alpine", + "Vulnerabilities": [ + { + "VulnerabilityID": "CVE-2019-14697", + "PkgID": "musl@1.1.18-r3", + "PkgName": "musl", + "InstalledVersion": "1.1.18-r3", + "FixedVersion": "1.1.18-r4", + "Layer": { + "Digest": "sha256:5d20c808ce198565ff70b3ed23a991dd49afac45dece63474b27ce6ed036adc6", + "DiffID": "sha256:3fc64803ca2de7279269048fe2b8b3c73d4536448c87c32375b2639ac168a48b" + }, + "SeveritySource": "nvd", + "PrimaryURL": "https://avd.aquasec.com/nvd/cve-2019-14697", + "DataSource": { + "ID": "alpine", + "Name": "Alpine Secdb", + "URL": "https://secdb.alpinelinux.org/" + }, + "Title": "musl libc through 1.1.23 has an x87 floating-point stack adjustment im ...", + "Description": "musl libc through 1.1.23 has an x87 floating-point stack adjustment imbalance, related to the math/i386/ directory. In some cases, use of this library could introduce out-of-bounds writes that are not present in an application's source code.", + "Severity": "CRITICAL", + "CweIDs": [ + "CWE-787" + ], + "CVSS": { + "nvd": { + "V2Vector": "AV:N/AC:L/Au:N/C:P/I:P/A:P", + "V3Vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "V2Score": 7.5, + "V3Score": 9.8 + } + }, + "References": [ + "http://www.openwall.com/lists/oss-security/2019/08/06/4", + "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-14697", + "https://git.musl-libc.org/cgit/musl/patch/?id=6818c31c9bc4bbad5357f1de14bedf781e5b349e", + "https://git.musl-libc.org/cgit/musl/patch/?id=f3ed8bfe8a82af1870ddc8696ed4cc1d5aa6b441", + "https://security.gentoo.org/glsa/202003-13", + "https://ubuntu.com/security/notices/USN-5990-1", + "https://www.openwall.com/lists/musl/2019/08/06/1", + "https://www.openwall.com/lists/oss-security/2019/08/06/1" + ], + "PublishedDate": "2019-08-06T16:15:00Z", + "LastModifiedDate": "2023-03-03T17:43:00Z" + }, + { + "VulnerabilityID": "CVE-2019-14697", + "PkgID": "musl-utils@1.1.18-r3", + "PkgName": "musl-utils", + "InstalledVersion": "1.1.18-r3", + "FixedVersion": "1.1.18-r4", + "Layer": { + "Digest": "sha256:5d20c808ce198565ff70b3ed23a991dd49afac45dece63474b27ce6ed036adc6", + "DiffID": "sha256:3fc64803ca2de7279269048fe2b8b3c73d4536448c87c32375b2639ac168a48b" + }, + "SeveritySource": "nvd", + "PrimaryURL": "https://avd.aquasec.com/nvd/cve-2019-14697", + "DataSource": { + "ID": "alpine", + "Name": "Alpine Secdb", + "URL": "https://secdb.alpinelinux.org/" + }, + "Title": "musl libc through 1.1.23 has an x87 floating-point stack adjustment im ...", + "Description": "musl libc through 1.1.23 has an x87 floating-point stack adjustment imbalance, related to the math/i386/ directory. In some cases, use of this library could introduce out-of-bounds writes that are not present in an application's source code.", + "Severity": "CRITICAL", + "CweIDs": [ + "CWE-787" + ], + "CVSS": { + "nvd": { + "V2Vector": "AV:N/AC:L/Au:N/C:P/I:P/A:P", + "V3Vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "V2Score": 7.5, + "V3Score": 9.8 + } + }, + "References": [ + "http://www.openwall.com/lists/oss-security/2019/08/06/4", + "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-14697", + "https://git.musl-libc.org/cgit/musl/patch/?id=6818c31c9bc4bbad5357f1de14bedf781e5b349e", + "https://git.musl-libc.org/cgit/musl/patch/?id=f3ed8bfe8a82af1870ddc8696ed4cc1d5aa6b441", + "https://security.gentoo.org/glsa/202003-13", + "https://ubuntu.com/security/notices/USN-5990-1", + "https://www.openwall.com/lists/musl/2019/08/06/1", + "https://www.openwall.com/lists/oss-security/2019/08/06/1" + ], + "PublishedDate": "2019-08-06T16:15:00Z", + "LastModifiedDate": "2023-03-03T17:43:00Z" + } + ] + } + ] + } + \ No newline at end of file diff --git a/tests/test_docker_client.py b/tests/test_docker_client.py new file mode 100644 index 0000000..55f50ad --- /dev/null +++ b/tests/test_docker_client.py @@ -0,0 +1,174 @@ +"""Docker Client Tests""" + +from unittest import TestCase +from unittest.mock import patch, MagicMock +from typing import Any, Optional +from json import dumps +from http.client import HTTPException +from docker_registry_client import DockerRegistryClient + + +class FakeHTTPSConnection: + """FakeHTTPSConnection Class""" + + def __init__(self, status: int = 200, headers: Optional[dict[str, str]] = None): + self.status = status + self.headers = {} if not headers else headers + + def request(self, url: str, **_): + """FakeHTTPSConnection Request Method""" + + self.url = url # pylint: disable=attribute-defined-outside-init + + def getresponse(self): + """FakeHTTPSConnection getresponse Method""" + + return FakeHTTPResponse(url=self.url, status=self.status, headers=self.headers) + + +class FakeHTTPResponse: # pylint: disable=too-few-public-methods + """FakeHTTPResponse Class""" + + def __init__(self, url: str, status: int, headers: dict[str, str]): + self.status = status + self.url = url + self.headers = headers + + if status == 200: + self.reason = "OK" + if status == 301: + if headers.get("location") == "/v2/_catalog": + self.status = 200 + self.reason = "OK" + self.reason = "Moved Permenatly" + if status == 404: + self.reason = "Not Found" + + def __to_json(self, obj: Any) -> str: + return dumps(obj=obj) + + def getheader(self, name: str) -> Optional[str]: + """FakeHTTPResponse Get Header Method""" + return self.headers.get(name) + + def read(self) -> dict[str, Any]: + """FakeHTTPResponse Read Method""" + + # Get Docker images + if "/v2/_catalog" in self.url: + return self.__to_json( + obj={"repositories": ["fake-alpine", "fake-ubuntu", "fake-python"]} + ) + + # Get Docker image tags + if "/tags/list" in self.url: + tags: list[str] = ["a", "b", "c"] + _, image_name, *_ = self.url.lstrip("/").split("/") + print(image_name) + if image_name == "fake-no-tags": + tags *= 0 + return self.__to_json(obj={"name": image_name, "tags": tags}) + + return self.__to_json(obj={}) + + +class DockerRegistryClientTests(TestCase): + """Docker Client Tests Class""" + + my_registry_host: str = "docker.registry.example.com" + my_registry_port: int = 12345 + my_registry_path: str = "/my/awesome/path" + my_registry_url: str = ( + f"https://{my_registry_host}:{my_registry_port}{my_registry_path}" + ) + my_ca_file: str = "./tests/certs/example.com.crt" + my_docker_registry_client: DockerRegistryClient = DockerRegistryClient( + registry_url=my_registry_url, ca_file=my_ca_file + ) + + def test_init(self): + """Docker Client Initialization Test""" + + self.assertEqual( + first=self.my_docker_registry_client.registry_host, + second=self.my_registry_host, + ) + self.assertEqual( + first=self.my_docker_registry_client.registry_port, + second=self.my_registry_port, + ) + self.assertEqual( + first=self.my_docker_registry_client.registry_path, + second=self.my_registry_path, + ) + with self.assertRaises(expected_exception=ValueError): + DockerRegistryClient( + registry_url="http://insecure.registry.example.com:8080" + ) + + @patch( + target="docker_registry_client.HTTPSConnection", + new=MagicMock(return_value=FakeHTTPSConnection()), + ) + def test_get_images(self): + """Docker Client Get Images Test""" + + my_fake_docker_registry_client: DockerRegistryClient = DockerRegistryClient( + registry_url="https://fake.registry.example.com:12345" + ) + self.assertEqual( + first=my_fake_docker_registry_client.get_images(), + second=["fake-alpine", "fake-ubuntu", "fake-python"], + ) + + @patch( + target="docker_registry_client.HTTPSConnection", + new=MagicMock(return_value=FakeHTTPSConnection()), + ) + def test_get_image_tags(self): + """Docker Client Get Image Tags Test""" + + my_fake_docker_registry_client: DockerRegistryClient = DockerRegistryClient( + registry_url="https://fake.registry.example.com:12345" + ) + self.assertEqual( + first=my_fake_docker_registry_client.get_image_tags(image="fake-no-tags"), + second=[], + ) + self.assertEqual( + first=my_fake_docker_registry_client.get_image_tags(image="fake-alpine"), + second=["fake-alpine:a", "fake-alpine:b", "fake-alpine:c"], + ) + + @patch( + target="docker_registry_client.HTTPSConnection", + new=MagicMock(return_value=FakeHTTPSConnection(status=404)), + ) + def test_request_404(self): + """Docker Client Get Image Tags Image Not Found (404)""" + + my_fake_docker_registry_client: DockerRegistryClient = DockerRegistryClient( + registry_url="https://fake.registry.example.com:12345" + ) + with self.assertRaises(expected_exception=HTTPException): + my_fake_docker_registry_client.get_image_tags(image="pouet") + + @patch( + target="docker_registry_client.HTTPSConnection", + new=MagicMock( + return_value=FakeHTTPSConnection( + status=301, headers={"location": "/v2/_catalog"} + ) + ), + ) + def test_request_301(self): + """Docker Client Get Images (301)""" + + my_fake_docker_registry_client: DockerRegistryClient = DockerRegistryClient( + registry_url="https://fake.registry.example.com:12345/" + ) + + self.assertEqual( + first=my_fake_docker_registry_client.get_images(), + second=["fake-alpine", "fake-ubuntu", "fake-python"], + ) diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 0000000..0dd2168 --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,52 @@ +"""Main Tests""" + +from unittest import TestCase +from json import dumps +from typing import Any +from main import export_scan_results, display_results + + +class MainTests(TestCase): + """Main Tests Class""" + + def test_display_results(self): + """Test Display Results Function""" + + results: dict[str, Any] = { + "fake-docker-registry.example.com/path/to/fake-alpine:123": { + "status": "OK" + }, + "fake-docker-registry.example.com/path/to/fake-alpine:456": { + "status": "NOK", + "vulnerabilities": {"summary": {"HIGH": 5}}, + }, + } + + results_list: list[str] = [ + "🟢 OK\tfake-docker-registry.example.com/path/to/fake-alpine:123", + "🔴 NOK\tfake-docker-registry.example.com/path/to/fake-alpine:456" + " ({'HIGH': 5})", + ] + + with self.assertLogs(level="INFO") as logging_watcher: + display_results(results=results) + for result_list in results_list: + self.assertIn( + member=result_list, + container=[ + ":".join(str(x).split(":")[2:]) for x in logging_watcher.output + ], + ) + + def test_export_scan_results(self): + """Test Export Scan Results Function""" + + my_output_file: str = "/tmp/results.json" + my_dict: dict[str, str] = {"Hello": "Wolrd"} + + export_scan_results(scan_results=my_dict, output_file=my_output_file) + + with open(file=my_output_file, encoding="UTF-8") as my_file: + self.assertEqual( + first="".join(my_file.readlines()), second=dumps(obj=my_dict, indent=2) + ) diff --git a/tests/test_scanner.py b/tests/test_scanner.py new file mode 100644 index 0000000..f6c5032 --- /dev/null +++ b/tests/test_scanner.py @@ -0,0 +1,96 @@ +"""Scanner Tests""" + +from unittest import TestCase +from unittest.mock import patch, MagicMock +from typing import Any +from dataclasses import dataclass +from scanner import Scanner, download_database +from scanner_options import ScannerOptions + + +@dataclass +class FakeCompletedProcess: + """FakeCompletedProcess Class""" + + stdout: bytes = b"" + returncode: int = 0 + + +class ScannerTests(TestCase): + """Scanner Tests Class""" + + my_docker_registry: str = "docker-registry.example.com:12345" + my_image_tags: list[str] = ["alpine:3.7"] + my_severity: str = "HIGH,CRITICAL" + my_scanners: str = "vuln,secret" + my_scanner_options: ScannerOptions = ScannerOptions( + severity=my_severity, scanners=my_scanners, processes=2 + ) + my_scanner: Scanner = Scanner( + docker_registry=my_docker_registry, + image_tags=my_image_tags, + options=my_scanner_options, + ) + with open( + file="./tests/fixtures/trivy_scan_result.json", + encoding="UTF-8", + ) as my_file: + run_scan_stdout: bytes = "".join(my_file.readlines()) + run_scan_return: dict[str, Any] = { + f"{my_docker_registry}/{my_image_tags[0]}": { + "status": "NOK", + "created": "2019-03-07T22:19:53.447205048Z", + "id": ( + "sha256:" + "6d1ef012b5674ad8a127ecfa9b5e6f5178d171b90ee462846974177fd9bdd39f" + ), + "labels": None, + "vulnerabilities": { + f"os-pkgs ({my_docker_registry}/{my_image_tags[0]}" + " (alpine 3.7.3))": [ + "CVE-2019-14697 (CRITICAL)", + ], + "summary": {"HIGH": 0, "CRITICAL": 1}, + }, + } + } + + def test_init(self): + """Test Scanner Initialization""" + + self.assertEqual( + first=self.my_scanner.docker_registry, second=self.my_docker_registry + ) + self.assertEqual(first=self.my_scanner.image_tags, second=self.my_image_tags) + self.assertEqual(first=self.my_scanner.severity, second=self.my_severity) + self.assertEqual(first=self.my_scanner.scanners, second=self.my_scanners) + + @patch( + target="scanner.run", + new=MagicMock( + return_value=FakeCompletedProcess(stdout=run_scan_stdout, returncode=111) + ), + ) + def test_run_scan(self): + """Test Run Scan Method""" + self.assertEqual( + first=self.my_scanner.run_scan(image_tag="alpine:3.7"), + second=self.run_scan_return, + ) + + @patch(target="scanner.run", new=MagicMock(return_value=FakeCompletedProcess())) + def test_download_database(self): + """Test Download Database""" + + download_database() + download_database(database="java") + + @patch( + target="scanner.run", + new=MagicMock(return_value=FakeCompletedProcess(returncode=1)), + ) + def test_download_database_raises(self): + """Test Download Database""" + + with self.assertRaises(expected_exception=SystemExit): + download_database() diff --git a/tests/test_scanner_options.py b/tests/test_scanner_options.py new file mode 100644 index 0000000..29cf20e --- /dev/null +++ b/tests/test_scanner_options.py @@ -0,0 +1,30 @@ +"""ScannerOptionsTests""" + +from unittest import TestCase +from scanner_options import ScannerOptions + + +class ScannerOptionsTests(TestCase): + """ScannerOptions Test Cases""" + + def test_min_severity(self): + """Tests min_severity""" + + self.assertEqual( + first=ScannerOptions(min_severity="UNKNOWN").severity, + second="UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL", + ) + self.assertEqual( + first=ScannerOptions(min_severity="MEDIUM").severity, + second="MEDIUM,HIGH,CRITICAL", + ) + self.assertEqual( + first=ScannerOptions(min_severity="LOW").severity, + second="LOW,MEDIUM,HIGH,CRITICAL", + ) + self.assertEqual( + first=ScannerOptions(min_severity="HIGH").severity, second="HIGH,CRITICAL" + ) + self.assertEqual( + first=ScannerOptions(min_severity="CRITICAL").severity, second="CRITICAL" + )