diff --git a/.github/workflows/schemacode_ci.yml b/.github/workflows/schemacode_ci.yml index cff7d01d54..5998813256 100644 --- a/.github/workflows/schemacode_ci.yml +++ b/.github/workflows/schemacode_ci.yml @@ -4,10 +4,13 @@ on: push: branches: - "master" + - "maint/*" tags: - "schema-*" pull_request: branches: + - "master" + - "maint/*" - "*" concurrency: @@ -36,7 +39,9 @@ jobs: run: pip install --upgrade tools/schemacode[all] if: ${{ startsWith(github.ref, 'refs/tags/schema-') }} - name: "Build archive on tag" - run: pytest tools/schemacode/bidsschematools -k make_archive + run: | + python -m pytest -k make_archive + working-directory: tools/schemacode env: BIDSSCHEMATOOLS_RELEASE: 1 if: ${{ startsWith(github.ref, 'refs/tags/schema-') }} @@ -87,14 +92,15 @@ jobs: - name: "Run tests" run: | - python -m pytest -vs --pyargs bidsschematools -m "not validate_schema" \ - --cov-append --cov-report=xml --cov=bidsschematools --doctest-modules + python -m pytest -vs --doctest-modules -m "not validate_schema" \ + --cov-append --cov-report=xml --cov-report=term --cov=src/bidsschematools + working-directory: tools/schemacode - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: unit_${{ matrix.os }}_${{ matrix.python-version }} - path: coverage.xml + path: tools/schemacode/coverage.xml if: success() publish: @@ -143,13 +149,16 @@ jobs: python -m pip install -e ./tools/schemacode[all] - name: Run schema validation tests - run: python -m pytest --pyargs bidsschematools -m "validate_schema" --cov-append --cov-report=xml --cov=bidsschematools + run: | + python -m pytest -vs --doctest-modules -m "not validate_schema" \ + --cov-append --cov-report=xml --cov-report=term --cov=src/bidsschematools + working-directory: tools/schemacode - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: schema_validation - path: coverage.xml + path: tools/schemacode/coverage.xml if: success() upload_to_codecov: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 83f40fdbb4..60c275dcf8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks ci: skip: [shellcheck] -exclude: 'tools/schemacode/bidsschematools/tests/data/broken_dataset_description.json' +exclude: 'tools/schemacode/src/bidsschematools/tests/data/broken_dataset_description.json' repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 @@ -89,7 +89,7 @@ repos: - types-jsonschema - jsonschema - httpx - args: ["tools/schemacode/bidsschematools"] + args: ["tools/schemacode/src"] pass_filenames: false - repo: https://github.com/koalaman/shellcheck-precommit rev: v0.10.0 diff --git a/tools/schemacode/MANIFEST.in b/tools/schemacode/MANIFEST.in new file mode 100644 index 0000000000..37f630a4ae --- /dev/null +++ b/tools/schemacode/MANIFEST.in @@ -0,0 +1 @@ +recursive-include tests * diff --git a/tools/schemacode/bidsschematools/__main__.py b/tools/schemacode/bidsschematools/__main__.py deleted file mode 100644 index a250e37cab..0000000000 --- a/tools/schemacode/bidsschematools/__main__.py +++ /dev/null @@ -1,57 +0,0 @@ -import logging -import os -import sys - -import click - -if sys.version_info < (3, 9): - from importlib_resources import files -else: - from importlib.resources import files - - -from .schema import export_schema, load_schema - - -@click.group() -@click.option("-v", "--verbose", count=True) -def cli(verbose): - """BIDS Schema Tools""" - logging.getLogger("bidsschematools").setLevel(logging.INFO - verbose * 10) - - -@cli.command() -@click.option("--schema") -@click.option("--output", default="-") -@click.pass_context -def export(ctx, schema, output): - """Export BIDS schema to JSON document""" - logger = logging.getLogger("bidsschematools") - schema = load_schema(schema) - text = export_schema(schema) - if output == "-": - logger.debug("Writing to stdout") - print(text) - else: - output = os.path.abspath(output) - logger.debug(f"Writing to {output}") - with open(output, "w") as fobj: - fobj.write(text) - - -@cli.command() -@click.option("--output", default="-") -@click.pass_context -def export_metaschema(ctx, output): - """Export BIDS schema to JSON document""" - metaschema = files("bidsschematools.data").joinpath("metaschema.json").read_text() - if output == "-": - print(metaschema, end="") - else: - output = os.path.abspath(output) - with open(output, "w") as fobj: - fobj.write(metaschema) - - -if __name__ == "__main__": - cli() diff --git a/tools/schemacode/bidsschematools/data/metaschema.json b/tools/schemacode/bidsschematools/data/metaschema.json deleted file mode 120000 index ae3cfc38f9..0000000000 --- a/tools/schemacode/bidsschematools/data/metaschema.json +++ /dev/null @@ -1 +0,0 @@ -../../../../src/metaschema.json \ No newline at end of file diff --git a/tools/schemacode/bidsschematools/data/schema b/tools/schemacode/bidsschematools/data/schema deleted file mode 120000 index a06955547a..0000000000 --- a/tools/schemacode/bidsschematools/data/schema +++ /dev/null @@ -1 +0,0 @@ -../../../../src/schema \ No newline at end of file diff --git a/tools/schemacode/pyproject.toml b/tools/schemacode/pyproject.toml index 9068e6d471..0d9a03ae72 100644 --- a/tools/schemacode/pyproject.toml +++ b/tools/schemacode/pyproject.toml @@ -60,21 +60,17 @@ bst = "bidsschematools.__main__:cli" Homepage = "https://github.com/bids-standard/bids-specification" [tool.setuptools.dynamic] -version = {file = "bidsschematools/data/schema/SCHEMA_VERSION"} +version = {file = "src/bidsschematools/data/schema/SCHEMA_VERSION"} [tool.setuptools.package-data] bidsschematools = [ "data/metaschema.json", "data/schema/BIDS_VERSION", - "data/schema/SCHEMA_VERSIO", + "data/schema/SCHEMA_VERSION", "data/schema/**/*.yaml", - "tests/data/**/*", - "tests/data/**/.bidsignore" + "tests/data/*" ] -[tool.setuptools.packages.find] -include = ["bidsschematools*"] - [tool.black] line-length = 99 include = '\.pyi?$' @@ -93,13 +89,6 @@ exclude = ''' ) ''' -[tool.coverage.run] -omit = [ - "*/*/tests/*", - "**/tests/*" -] -parallel = true - [tool.isort] multi_line_output = 3 profile = "black" @@ -113,3 +102,12 @@ markers = [ ] minversion = "6.0" xfail_strict = true + +[tool.coverage.paths] +source = [ + "src/bidsschematools", + "**/site-packages/bidsschematools" +] + +[tool.coverage.run] +parallel = true diff --git a/tools/schemacode/bidsschematools/__init__.py b/tools/schemacode/src/bidsschematools/__init__.py similarity index 100% rename from tools/schemacode/bidsschematools/__init__.py rename to tools/schemacode/src/bidsschematools/__init__.py diff --git a/tools/schemacode/src/bidsschematools/__main__.py b/tools/schemacode/src/bidsschematools/__main__.py new file mode 100644 index 0000000000..777bea2d87 --- /dev/null +++ b/tools/schemacode/src/bidsschematools/__main__.py @@ -0,0 +1,189 @@ +import json +import logging +import os +import re +import sys +from itertools import chain + +import click + +if sys.version_info < (3, 9): + from importlib_resources import files +else: + from importlib.resources import files + +from .rules import regexify_filename_rules +from .schema import export_schema, load_schema +from .validator import _bidsignore_check + + +@click.group() +@click.option("-v", "--verbose", count=True) +@click.option("-q", "--quiet", count=True) +def cli(verbose, quiet): + """BIDS Schema Tools""" + verbose = verbose - quiet + logging.getLogger("bidsschematools").setLevel(logging.WARNING - verbose * 10) + + +@cli.command() +@click.option("--schema") +@click.option("--output", default="-") +@click.pass_context +def export(ctx, schema, output): + """Export BIDS schema to JSON document""" + logger = logging.getLogger("bidsschematools") + schema = load_schema(schema) + text = export_schema(schema) + if output == "-": + logger.debug("Writing to stdout") + print(text) + else: + output = os.path.abspath(output) + logger.debug(f"Writing to {output}") + with open(output, "w") as fobj: + fobj.write(text) + + +@cli.command() +@click.option("--output", default="-") +@click.pass_context +def export_metaschema(ctx, output): + """Export BIDS schema to JSON document""" + metaschema = files("bidsschematools.data").joinpath("metaschema.json").read_text() + if output == "-": + print(metaschema, end="") + else: + output = os.path.abspath(output) + with open(output, "w") as fobj: + fobj.write(metaschema) + + +@cli.command("pre-receive-hook") +@click.option("--schema", "-s", type=click.Path(), help="Path to the BIDS schema") +@click.option( + "--input", "-i", "input_", default="-", type=click.Path(), help="Input file (default: stdin)" +) +@click.option( + "--output", + "-o", + "output", + default="-", + type=click.Path(), + help="Output file (default: stdout)", +) +def pre_receive_hook(schema, input_, output): + """Validate filenames from a list of files against the BIDS schema + + The expected input takes the following form: + + ``` + bids-hook-v2 + {"Name": "My dataset", "BIDSVersion": "1.9.0", "DatasetType": "raw"} + ignore-pattern1 + ... + ignore-patternN + 0001 + .datalad/config + .gitattributes + CHANGES + README + dataset_description.json + participants.tsv + sub-01/anat/sub-01_T1w.nii.gz + ... + ``` + + The header identifies the protocol version. For protocol ``bids-hook-v2``, + the second line MUST be the dataset_description.json file, with any newlines removed. + The following lines, up to the line containing "0001", are ignore patterns + from the .bidsignore file. The lines following "0001" are the filenames to + be validated. + + This is intended to be used in a git pre-receive hook. + """ + logger = logging.getLogger("bidsschematools") + schema = load_schema(schema) + + # Slurp inputs for now; we can think about streaming later + if input_ == "-": + stream = sys.stdin + else: + stream = open(input_) + + first_line = next(stream) + if first_line == "bids-hook-v2\n": + # V2 format: header line, description JSON, followed by legacy format + description_str = next(stream) + fail = False + try: + description: dict = json.loads(description_str) + except json.JSONDecodeError: + fail = True + if fail or not isinstance(description, dict): + logger.critical("Protocol error: invalid JSON in description") + logger.critical( + "Dataset description must be one JSON object, written to a single line" + ) + logger.critical("Received: %s", description_str) + stream.close() + sys.exit(2) + else: + # Legacy: ignore patterns, followed by "0001", followed by filenames + stream = chain([first_line], stream) + description = {} + + dataset_type = description.get("DatasetType", "raw") + logger.info("Dataset type: %s", dataset_type) + + ignore = [] + for line in stream: + if line == "0001\n": + break + ignore.append(line.strip()) + logger.info("Ignore patterns found: %d", len(ignore)) + + all_rules = chain.from_iterable( + regexify_filename_rules(group, schema, level=2) + for group in (schema.rules.files.common, schema.rules.files.raw) + ) + if dataset_type == "derivative": + all_rules = chain( + all_rules, + regexify_filename_rules(schema.rules.files.derivatives, schema, level=2), + ) + + regexes = [rule["regex"] for rule in all_rules] + # XXX Hack for phenotype files - this can be removed once we + # have a schema definition for them + regexes.append(r"phenotype/.*\.(tsv|json)") + + output = sys.stdout if output == "-" else open(output, "w") + + rc = 0 + any_files = False + valid_files = 0 + with output: + for filename in stream: + if not any_files: + logger.debug("Validating files, first file: %s", filename) + any_files = True + filename = filename.strip() + if any(_bidsignore_check(pattern, filename, "") for pattern in ignore): + continue + if not any(re.match(regex, filename) for regex in regexes): + print(filename, file=output) + rc = 1 + else: + valid_files += 1 + + if valid_files == 0: + logger.error("No files to validate") + rc = 2 + + stream.close() + sys.exit(rc) + + +if __name__ == "__main__": + cli() diff --git a/tools/schemacode/bidsschematools/conftest.py b/tools/schemacode/src/bidsschematools/conftest.py similarity index 82% rename from tools/schemacode/bidsschematools/conftest.py rename to tools/schemacode/src/bidsschematools/conftest.py index 2b87854a99..0a47d26535 100644 --- a/tools/schemacode/bidsschematools/conftest.py +++ b/tools/schemacode/src/bidsschematools/conftest.py @@ -1,6 +1,6 @@ import logging import tempfile -from importlib.resources import as_file, files +from pathlib import Path from subprocess import run import pytest @@ -31,23 +31,37 @@ ] +@pytest.fixture(scope="session") +def tests_data_dir(): + try: + this_file = Path(__file__) + except NameError: + return None + + data_dir = this_file.parent.parent.parent / "tests" / "data" + + if data_dir.exists(): + return data_dir + + def get_gitrepo_fixture(url, whitelist): @pytest.fixture(scope="session") - def fixture(): + def fixture(tests_data_dir): + if tests_data_dir is None: + pytest.skip("No test data directory found; probably in an installed package") archive_name = url.rsplit("/", 1)[-1] - testdata_dir = files("bidsschematools.tests.data") / archive_name - if testdata_dir.is_dir(): + archive_dir = tests_data_dir / archive_name + if archive_dir.is_dir(): lgr.info( - f"Found static testdata archive under `{testdata_dir}`. " + f"Found static testdata archive under `{archive_dir}`. " "Not downloading latest data from version control." ) - with as_file(testdata_dir) as path: - yield path + yield archive_dir else: lgr.info( "No static testdata available under `%s`. " "Attempting to fetch live data from version control.", - testdata_dir, + archive_dir, ) with tempfile.TemporaryDirectory() as path: lgr.debug("Cloning %r into %r", url, path) diff --git a/tools/schemacode/bidsschematools/data/__init__.py b/tools/schemacode/src/bidsschematools/data/__init__.py similarity index 100% rename from tools/schemacode/bidsschematools/data/__init__.py rename to tools/schemacode/src/bidsschematools/data/__init__.py diff --git a/tools/schemacode/src/bidsschematools/data/metaschema.json b/tools/schemacode/src/bidsschematools/data/metaschema.json new file mode 120000 index 0000000000..399e1beffb --- /dev/null +++ b/tools/schemacode/src/bidsschematools/data/metaschema.json @@ -0,0 +1 @@ +../../../../../src/metaschema.json \ No newline at end of file diff --git a/tools/schemacode/src/bidsschematools/data/schema b/tools/schemacode/src/bidsschematools/data/schema new file mode 120000 index 0000000000..1e33d0b966 --- /dev/null +++ b/tools/schemacode/src/bidsschematools/data/schema @@ -0,0 +1 @@ +../../../../../src/schema \ No newline at end of file diff --git a/tools/schemacode/bidsschematools/data/tests/test_rules.py b/tools/schemacode/src/bidsschematools/data/tests/test_rules.py similarity index 100% rename from tools/schemacode/bidsschematools/data/tests/test_rules.py rename to tools/schemacode/src/bidsschematools/data/tests/test_rules.py diff --git a/tools/schemacode/bidsschematools/expressions.py b/tools/schemacode/src/bidsschematools/expressions.py similarity index 100% rename from tools/schemacode/bidsschematools/expressions.py rename to tools/schemacode/src/bidsschematools/expressions.py diff --git a/tools/schemacode/bidsschematools/render/__init__.py b/tools/schemacode/src/bidsschematools/render/__init__.py similarity index 100% rename from tools/schemacode/bidsschematools/render/__init__.py rename to tools/schemacode/src/bidsschematools/render/__init__.py diff --git a/tools/schemacode/bidsschematools/render/tables.py b/tools/schemacode/src/bidsschematools/render/tables.py similarity index 100% rename from tools/schemacode/bidsschematools/render/tables.py rename to tools/schemacode/src/bidsschematools/render/tables.py diff --git a/tools/schemacode/bidsschematools/render/text.py b/tools/schemacode/src/bidsschematools/render/text.py similarity index 100% rename from tools/schemacode/bidsschematools/render/text.py rename to tools/schemacode/src/bidsschematools/render/text.py diff --git a/tools/schemacode/bidsschematools/render/utils.py b/tools/schemacode/src/bidsschematools/render/utils.py similarity index 100% rename from tools/schemacode/bidsschematools/render/utils.py rename to tools/schemacode/src/bidsschematools/render/utils.py diff --git a/tools/schemacode/bidsschematools/rules.py b/tools/schemacode/src/bidsschematools/rules.py similarity index 100% rename from tools/schemacode/bidsschematools/rules.py rename to tools/schemacode/src/bidsschematools/rules.py diff --git a/tools/schemacode/bidsschematools/schema.py b/tools/schemacode/src/bidsschematools/schema.py similarity index 100% rename from tools/schemacode/bidsschematools/schema.py rename to tools/schemacode/src/bidsschematools/schema.py diff --git a/tools/schemacode/bidsschematools/tests/__init__.py b/tools/schemacode/src/bidsschematools/tests/__init__.py similarity index 100% rename from tools/schemacode/bidsschematools/tests/__init__.py rename to tools/schemacode/src/bidsschematools/tests/__init__.py diff --git a/tools/schemacode/bidsschematools/tests/data/__init__.py b/tools/schemacode/src/bidsschematools/tests/data/__init__.py similarity index 100% rename from tools/schemacode/bidsschematools/tests/data/__init__.py rename to tools/schemacode/src/bidsschematools/tests/data/__init__.py diff --git a/tools/schemacode/bidsschematools/tests/data/broken_dataset_description.json b/tools/schemacode/src/bidsschematools/tests/data/broken_dataset_description.json similarity index 100% rename from tools/schemacode/bidsschematools/tests/data/broken_dataset_description.json rename to tools/schemacode/src/bidsschematools/tests/data/broken_dataset_description.json diff --git a/tools/schemacode/bidsschematools/tests/data/expected_bids_validator_xs_write.log b/tools/schemacode/src/bidsschematools/tests/data/expected_bids_validator_xs_write.log similarity index 100% rename from tools/schemacode/bidsschematools/tests/data/expected_bids_validator_xs_write.log rename to tools/schemacode/src/bidsschematools/tests/data/expected_bids_validator_xs_write.log diff --git a/tools/schemacode/bidsschematools/tests/test_expressions.py b/tools/schemacode/src/bidsschematools/tests/test_expressions.py similarity index 100% rename from tools/schemacode/bidsschematools/tests/test_expressions.py rename to tools/schemacode/src/bidsschematools/tests/test_expressions.py diff --git a/tools/schemacode/bidsschematools/tests/test_make_testdata.py b/tools/schemacode/src/bidsschematools/tests/test_make_testdata.py similarity index 52% rename from tools/schemacode/bidsschematools/tests/test_make_testdata.py rename to tools/schemacode/src/bidsschematools/tests/test_make_testdata.py index 8ef6443d8d..77c6ae9d00 100644 --- a/tools/schemacode/bidsschematools/tests/test_make_testdata.py +++ b/tools/schemacode/src/bidsschematools/tests/test_make_testdata.py @@ -1,6 +1,5 @@ import os import shutil -from importlib.resources import files import pytest @@ -13,7 +12,7 @@ def require_env(var): @require_env("BIDSSCHEMATOOLS_RELEASE") -def test_make_archive(bids_examples, bids_error_examples): +def test_make_archive(tests_data_dir, bids_examples, bids_error_examples): """ ATTENTION! This is not a test! Create static testdata archive containing the bidsschematools data reference whitelist. @@ -30,15 +29,10 @@ def test_make_archive(bids_examples, bids_error_examples): testdata archive creation is now inconspicuously posing as a test. """ - testdata_dir = files("bidsschematools.tests.data") ignore_git = shutil.ignore_patterns(".git*") - shutil.copytree(bids_examples, testdata_dir / "bids-examples", ignore=ignore_git) - shutil.copytree(bids_error_examples, testdata_dir / "bids-error-examples", ignore=ignore_git) - - # Keeping this for now, it would be really nice to have a separate archive someday. - # archive_name = f"bidsschematools-testdata-{__version__}" - # archive_path = f"/tmp/{archive_name}.tar.gz" - - # with tarfile.open(archive_path, "w:gz") as tar: - # tar.add(bids_examples, arcname=f"{archive_name}/bids-examples") - # tar.add(bids_error_examples, arcname=f"{archive_name}/bids-error-examples") + target_examples = tests_data_dir / "bids-examples" + target_error_examples = tests_data_dir / "bids-error-examples" + if bids_examples != target_examples: + shutil.copytree(bids_examples, target_examples, ignore=ignore_git) + if bids_error_examples != target_error_examples: + shutil.copytree(bids_error_examples, target_error_examples, ignore=ignore_git) diff --git a/tools/schemacode/bidsschematools/tests/test_render_tables.py b/tools/schemacode/src/bidsschematools/tests/test_render_tables.py similarity index 100% rename from tools/schemacode/bidsschematools/tests/test_render_tables.py rename to tools/schemacode/src/bidsschematools/tests/test_render_tables.py diff --git a/tools/schemacode/bidsschematools/tests/test_render_text.py b/tools/schemacode/src/bidsschematools/tests/test_render_text.py similarity index 100% rename from tools/schemacode/bidsschematools/tests/test_render_text.py rename to tools/schemacode/src/bidsschematools/tests/test_render_text.py diff --git a/tools/schemacode/bidsschematools/tests/test_render_utils.py b/tools/schemacode/src/bidsschematools/tests/test_render_utils.py similarity index 100% rename from tools/schemacode/bidsschematools/tests/test_render_utils.py rename to tools/schemacode/src/bidsschematools/tests/test_render_utils.py diff --git a/tools/schemacode/bidsschematools/tests/test_rules.py b/tools/schemacode/src/bidsschematools/tests/test_rules.py similarity index 100% rename from tools/schemacode/bidsschematools/tests/test_rules.py rename to tools/schemacode/src/bidsschematools/tests/test_rules.py diff --git a/tools/schemacode/bidsschematools/tests/test_schema.py b/tools/schemacode/src/bidsschematools/tests/test_schema.py similarity index 100% rename from tools/schemacode/bidsschematools/tests/test_schema.py rename to tools/schemacode/src/bidsschematools/tests/test_schema.py diff --git a/tools/schemacode/bidsschematools/tests/test_validator.py b/tools/schemacode/src/bidsschematools/tests/test_validator.py similarity index 100% rename from tools/schemacode/bidsschematools/tests/test_validator.py rename to tools/schemacode/src/bidsschematools/tests/test_validator.py diff --git a/tools/schemacode/bidsschematools/types/__init__.py b/tools/schemacode/src/bidsschematools/types/__init__.py similarity index 100% rename from tools/schemacode/bidsschematools/types/__init__.py rename to tools/schemacode/src/bidsschematools/types/__init__.py diff --git a/tools/schemacode/bidsschematools/types/namespace.py b/tools/schemacode/src/bidsschematools/types/namespace.py similarity index 100% rename from tools/schemacode/bidsschematools/types/namespace.py rename to tools/schemacode/src/bidsschematools/types/namespace.py diff --git a/tools/schemacode/bidsschematools/utils.py b/tools/schemacode/src/bidsschematools/utils.py similarity index 93% rename from tools/schemacode/bidsschematools/utils.py rename to tools/schemacode/src/bidsschematools/utils.py index ec2947fb37..6f9850edb0 100644 --- a/tools/schemacode/bidsschematools/utils.py +++ b/tools/schemacode/src/bidsschematools/utils.py @@ -29,7 +29,7 @@ def get_logger(name=None): logging.Logger logger object. """ - return logging.getLogger("bids-schema" + (".%s" % name if name else "")) + return logging.getLogger("bidsschematools" + (".%s" % name if name else "")) def set_logger_level(lgr, level): diff --git a/tools/schemacode/bidsschematools/validator.py b/tools/schemacode/src/bidsschematools/validator.py similarity index 100% rename from tools/schemacode/bidsschematools/validator.py rename to tools/schemacode/src/bidsschematools/validator.py diff --git a/tools/schemacode/tests/data/.gitignore b/tools/schemacode/tests/data/.gitignore new file mode 100644 index 0000000000..2dc3af1b14 --- /dev/null +++ b/tools/schemacode/tests/data/.gitignore @@ -0,0 +1 @@ +bids*-examples