From d4e0c9bfaa803899a07639c0d07240bb73158cdc Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 28 Jun 2024 13:40:32 +0200 Subject: [PATCH 1/8] reduce input variants to only _used_ input variants before recomputing output metadata reduces get_used_vars time dramatically because input_variants can be several thousand items long to produce 10s of variants --- conda_smithy/configure_feedstock.py | 4 ++++ news/1968_input_variants.rst | 3 +++ 2 files changed, 7 insertions(+) create mode 100644 news/1968_input_variants.rst diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 34a307faf..ae1bb67f7 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -922,8 +922,12 @@ def _conda_build_api_render_for_smithy( permit_unsatisfiable_variants=permit_unsatisfiable_variants, ) output_metas = [] + # reduce input variant set to those that are actually used + input_variants = [meta.config.variant for meta, *_ in metadata_tuples] for meta, download, render_in_env in metadata_tuples: if not meta.skip() or not config.trim_skip: + # avoid computations on very large, redundant input_variants + meta.config.input_variants = input_variants for od, om in meta.get_output_metadata_set( permit_unsatisfiable_variants=permit_unsatisfiable_variants, permit_undefined_jinja=not finalize, diff --git a/news/1968_input_variants.rst b/news/1968_input_variants.rst new file mode 100644 index 000000000..859b399ba --- /dev/null +++ b/news/1968_input_variants.rst @@ -0,0 +1,3 @@ +**Fixed:** + +* Improved render time for large numbers of variants From c5a90cb5c93e6be6b7d94f6e2ec751d8634e3cbb Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 28 Jun 2024 17:44:17 +0200 Subject: [PATCH 2/8] reduce variants before passing to conda_render saves a lot of time in render_recipe --- conda_smithy/configure_feedstock.py | 50 +++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index ae1bb67f7..7c7a213e6 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -42,6 +42,7 @@ import conda_build.utils import conda_build.variants from conda_build import __version__ as conda_build_version +from conda_build.metadata import MetaData from jinja2 import FileSystemLoader from jinja2.sandbox import SandboxedEnvironment @@ -880,6 +881,46 @@ def migrate_combined_spec(combined_spec, forge_dir, config, forge_config): return combined_spec +def reduce_variants(recipe_path, config, variants): + """Subset of render_recipe to compute reduced variant matrix + + large numbers of unused variants greatly increase render time + """ + + # from render_recipe + with conda_build.render.open_recipe(recipe_path) as recipe: + metadata = MetaData(str(recipe), config=config) + + # from distribute_variants + # explode variants dict to list of variants + variants = initial_variants = conda_build.variants.get_package_variants( + metadata, variants=variants + ) + logger.debug(f"Starting with {len(initial_variants)} variants") + if metadata.noarch or metadata.noarch_python: + # filter variants by the newest Python version + version = sorted( + { + version + for variant in variants + if (version := variant.get("python")) + }, + key=lambda key: VersionOrder(key.split(" ")[0]), + )[-1] + variants = conda_build.variants.filter_by_key_value( + variants, "python", version, "noarch_python_reduction" + ) + metadata.config.variant = variants[0] + metadata.config.variants = variants + all_used = metadata.get_used_vars(force_global=True) + reduced_variants = metadata.get_reduced_variant_set(all_used) + logger.info(f"Rendering {len(reduced_variants)} variants") + # return reduced, exploded list of variant dict to input format + return conda_build.variants.list_of_dicts_to_dict_of_lists( + reduced_variants + ) + + def _conda_build_api_render_for_smithy( recipe_path, config=None, @@ -913,6 +954,12 @@ def _conda_build_api_render_for_smithy( config = get_or_merge_config(config, **kwargs) + # reduce unused variants first, they get very expensive in render_recipe + if variants: + variants = reduce_variants( + recipe_path, config=config, variants=variants + ) + metadata_tuples = render_recipe( recipe_path, bypass_env_check=bypass_env_check, @@ -923,11 +970,8 @@ def _conda_build_api_render_for_smithy( ) output_metas = [] # reduce input variant set to those that are actually used - input_variants = [meta.config.variant for meta, *_ in metadata_tuples] for meta, download, render_in_env in metadata_tuples: if not meta.skip() or not config.trim_skip: - # avoid computations on very large, redundant input_variants - meta.config.input_variants = input_variants for od, om in meta.get_output_metadata_set( permit_unsatisfiable_variants=permit_unsatisfiable_variants, permit_undefined_jinja=not finalize, From cd985dda66eb96e6ef15d2fe16b855695b887f51 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 28 Jun 2024 21:07:18 +0200 Subject: [PATCH 3/8] try a simpler reduction just remove unused variants that add dimensionality, don't try to remove anything else --- conda_smithy/configure_feedstock.py | 38 ++++++++++++----------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index ae9808b16..5e9f30bc8 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -879,7 +879,7 @@ def migrate_combined_spec(combined_spec, forge_dir, config, forge_config): return combined_spec -def reduce_variants(recipe_path, config, variants): +def reduce_variants(recipe_path, config, input_variants): """Subset of render_recipe to compute reduced variant matrix large numbers of unused variants greatly increase render time @@ -892,31 +892,25 @@ def reduce_variants(recipe_path, config, variants): # from distribute_variants # explode variants dict to list of variants variants = initial_variants = conda_build.variants.get_package_variants( - metadata, variants=variants + metadata, variants=input_variants ) logger.debug(f"Starting with {len(initial_variants)} variants") - if metadata.noarch or metadata.noarch_python: - # filter variants by the newest Python version - version = sorted( - { - version - for variant in variants - if (version := variant.get("python")) - }, - key=lambda key: VersionOrder(key.split(" ")[0]), - )[-1] - variants = conda_build.variants.filter_by_key_value( - variants, "python", version, "noarch_python_reduction" - ) metadata.config.variant = variants[0] metadata.config.variants = variants all_used = metadata.get_used_vars(force_global=True) - reduced_variants = metadata.get_reduced_variant_set(all_used) - logger.info(f"Rendering {len(reduced_variants)} variants") - # return reduced, exploded list of variant dict to input format - return conda_build.variants.list_of_dicts_to_dict_of_lists( - reduced_variants - ) + new_input_variants = input_variants.copy() + # trim unused dimensions, these cost a lot in render_recipe! + for key, value in input_variants.items(): + if len(value) > 1 and key not in all_used | { + "zip_keys", + "pin_run_as_build", + "ignore_build_only_deps", + "extend_keys", + }: + logger.debug(f"Trimming unused dimension: {key}") + new_input_variants.pop(key) + _trim_unused_zip_keys(new_input_variants) + return new_input_variants def _conda_build_api_render_for_smithy( @@ -955,7 +949,7 @@ def _conda_build_api_render_for_smithy( # reduce unused variants first, they get very expensive in render_recipe if variants: variants = reduce_variants( - recipe_path, config=config, variants=variants + recipe_path, config=config, input_variants=variants ) metadata_tuples = render_recipe( From ffc2a0707dac1fb957c35697d342a9e65d33b2d2 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 1 Jul 2024 09:22:44 +0200 Subject: [PATCH 4/8] top-level always_keep_keys since it's used in more than one place --- conda_smithy/configure_feedstock.py | 50 +++++++++++++++-------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 5e9f30bc8..c6278b892 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -90,6 +90,24 @@ os.environ.get("CONDA_FORGE_PINNING_LIFETIME", 15 * 60) ) +ALWAYS_KEEP_KEYS = { + "zip_keys", + "pin_run_as_build", + "MACOSX_DEPLOYMENT_TARGET", + "MACOSX_SDK_VERSION", + "macos_min_version", + "macos_machine", + "channel_sources", + "channel_targets", + "docker_image", + "build_number_decrement", + # The following keys are required for some of our aarch64 builds + # Added in https://github.com/conda-forge/conda-forge-pinning-feedstock/pull/180 + "cdt_arch", + "cdt_name", + "BUILD", +} + # use lru_cache to avoid repeating warnings endlessly; # this keeps track of 10 different messages and then warns again @@ -602,23 +620,7 @@ def _collapse_subpackage_variants( ) # Add in some variables that should always be preserved - always_keep_keys = { - "zip_keys", - "pin_run_as_build", - "MACOSX_DEPLOYMENT_TARGET", - "MACOSX_SDK_VERSION", - "macos_min_version", - "macos_machine", - "channel_sources", - "channel_targets", - "docker_image", - "build_number_decrement", - # The following keys are required for some of our aarch64 builds - # Added in https://github.com/conda-forge/conda-forge-pinning-feedstock/pull/180 - "cdt_arch", - "cdt_name", - "BUILD", - } + always_keep_keys = set() | ALWAYS_KEEP_KEYS if not is_noarch: always_keep_keys.add("target_platform") @@ -898,15 +900,14 @@ def reduce_variants(recipe_path, config, input_variants): metadata.config.variant = variants[0] metadata.config.variants = variants all_used = metadata.get_used_vars(force_global=True) - new_input_variants = input_variants.copy() # trim unused dimensions, these cost a lot in render_recipe! + all_used.update(ALWAYS_KEEP_KEYS) + all_used.update( + {"target_platform", "extend_keys", "ignore_build_only_deps"} + ) + new_input_variants = input_variants.copy() for key, value in input_variants.items(): - if len(value) > 1 and key not in all_used | { - "zip_keys", - "pin_run_as_build", - "ignore_build_only_deps", - "extend_keys", - }: + if len(value) > 1 and key not in all_used: logger.debug(f"Trimming unused dimension: {key}") new_input_variants.pop(key) _trim_unused_zip_keys(new_input_variants) @@ -1103,6 +1104,7 @@ def _render_ci_provider( # CBC yaml files where variants in the migrators are not in the CBC. # Thus we move it out of the way. # TODO: upstream this as a flag in conda-build + logger.info(f"Getting variants for {platform}-{arch}") try: _recipe_cbc = os.path.join( forge_dir, From 15e4e0f589bd0cdf00a77814841d8283e5f3fa1c Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 1 Jul 2024 13:08:43 +0200 Subject: [PATCH 5/8] More comments in reduce_variants --- conda_smithy/configure_feedstock.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index c6278b892..c89f678fe 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -896,21 +896,31 @@ def reduce_variants(recipe_path, config, input_variants): variants = initial_variants = conda_build.variants.get_package_variants( metadata, variants=input_variants ) - logger.debug(f"Starting with {len(initial_variants)} variants") + logger.debug(f"Starting with {len(initial_variants)} input variants") metadata.config.variant = variants[0] metadata.config.variants = variants + + # force_global finds variables in the whole recipe, not just a single output. + # Without this, dependencies won't be found for multi-output recipes + # This may not be comprehensive! all_used = metadata.get_used_vars(force_global=True) # trim unused dimensions, these cost a lot in render_recipe! + # because `get_used_vars` above _may_ not catch all possible used variables, + # only trim unused variables that increase dimensionality. all_used.update(ALWAYS_KEEP_KEYS) all_used.update( {"target_platform", "extend_keys", "ignore_build_only_deps"} ) new_input_variants = input_variants.copy() for key, value in input_variants.items(): + # only consider keys that increase render dimensionality for trimming at this stage + # so we don't have to trust all_used to find _everything_ if len(value) > 1 and key not in all_used: logger.debug(f"Trimming unused dimension: {key}") new_input_variants.pop(key) _trim_unused_zip_keys(new_input_variants) + # new_variants = metadata.get_reduced_variant_set(all_used) + # logger.info(f"Rendering with {len(new_variants)} input variants") return new_input_variants From dc2c06260166df261c5cbaad5316b3c9e3b412b7 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 18 Sep 2024 08:47:57 +0200 Subject: [PATCH 6/8] logging lint --- conda_smithy/configure_feedstock.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 376488535..d637923c0 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -872,7 +872,7 @@ def reduce_variants(recipe_path, config, input_variants): variants = initial_variants = conda_build.variants.get_package_variants( metadata, variants=input_variants ) - logger.debug(f"Starting with {len(initial_variants)} input variants") + logger.debug("Starting with %s input variants", len(initial_variants)) metadata.config.variant = variants[0] metadata.config.variants = variants @@ -892,7 +892,7 @@ def reduce_variants(recipe_path, config, input_variants): # only consider keys that increase render dimensionality for trimming at this stage # so we don't have to trust all_used to find _everything_ if len(value) > 1 and key not in all_used: - logger.debug(f"Trimming unused dimension: {key}") + logger.debug("Trimming unused dimension: %s", key) new_input_variants.pop(key) _trim_unused_zip_keys(new_input_variants) # new_variants = metadata.get_reduced_variant_set(all_used) @@ -1116,7 +1116,7 @@ def _render_ci_provider( # CBC yaml files where variants in the migrators are not in the CBC. # Thus we move it out of the way. # TODO: upstream this as a flag in conda-build - logger.info(f"Getting variants for {platform}-{arch}") + logger.info("Getting variants for %s-%s", platform, arch) try: _recipe_cbc = os.path.join( forge_dir, From 33062752faa4fba881e184a5d0c5b4cb5bc55fb0 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 18 Sep 2024 15:13:07 +0200 Subject: [PATCH 7/8] capture used_vars in scripts when reducing variants --- conda_smithy/configure_feedstock.py | 16 ++++ .../recipes/multiple_outputs/build-output1.sh | 1 + .../recipes/multiple_outputs/build-output2.sh | 2 + .../multiple_outputs/conda_build_config.yaml | 12 +++ tests/recipes/multiple_outputs/meta.yaml | 2 + tests/test_configure_feedstock.py | 85 +++++++++++++------ 6 files changed, 92 insertions(+), 26 deletions(-) create mode 100644 tests/recipes/multiple_outputs/build-output1.sh create mode 100644 tests/recipes/multiple_outputs/build-output2.sh diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index d637923c0..fd925b63d 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -880,6 +880,22 @@ def reduce_variants(recipe_path, config, input_variants): # Without this, dependencies won't be found for multi-output recipes # This may not be comprehensive! all_used = metadata.get_used_vars(force_global=True) + # conservatively, find all used vars in all scripts + # we can't use `output.script` because these aren't computed yet + # bonus: this will find output.build.script, which conda-build does not. + for script in Path(recipe_path).glob("*.sh"): + all_used.update( + conda_build.variants.find_used_variables_in_shell_script( + metadata.config.variant, script + ) + ) + for script in Path(recipe_path).glob("*.bat"): + all_used.update( + conda_build.variants.find_used_variables_in_batch_script( + metadata.config.variant, script + ) + ) + # trim unused dimensions, these cost a lot in render_recipe! # because `get_used_vars` above _may_ not catch all possible used variables, # only trim unused variables that increase dimensionality. diff --git a/tests/recipes/multiple_outputs/build-output1.sh b/tests/recipes/multiple_outputs/build-output1.sh new file mode 100644 index 000000000..4a2464bfd --- /dev/null +++ b/tests/recipes/multiple_outputs/build-output1.sh @@ -0,0 +1 @@ +echo "${script_only}" diff --git a/tests/recipes/multiple_outputs/build-output2.sh b/tests/recipes/multiple_outputs/build-output2.sh new file mode 100644 index 000000000..dfece6418 --- /dev/null +++ b/tests/recipes/multiple_outputs/build-output2.sh @@ -0,0 +1,2 @@ +echo "${script_only}" +echo "${script_only_2}" diff --git a/tests/recipes/multiple_outputs/conda_build_config.yaml b/tests/recipes/multiple_outputs/conda_build_config.yaml index d0750ab2a..f13fd1550 100644 --- a/tests/recipes/multiple_outputs/conda_build_config.yaml +++ b/tests/recipes/multiple_outputs/conda_build_config.yaml @@ -24,6 +24,18 @@ zlib: nonexistent: - a - b +# not used; should not be picked up +unused: + - a + - b +# used in all output scripts +script_only: + - a + - b +# used in output2 script +script_only_2: + - a2 + - b2 # zip these just for testing purposes zip_keys: - diff --git a/tests/recipes/multiple_outputs/meta.yaml b/tests/recipes/multiple_outputs/meta.yaml index 81dc942a2..875af14e2 100644 --- a/tests/recipes/multiple_outputs/meta.yaml +++ b/tests/recipes/multiple_outputs/meta.yaml @@ -14,6 +14,7 @@ requirements: outputs: - name: test_output_1 + script: build-output1.sh requirements: - bzip2 test: @@ -21,6 +22,7 @@ outputs: - test -e $PREFIX/lib/libz.so # [linux] - test -e $PREFIX/lib/libz.dylib # [osx] - name: test_output_2 + script: build-output2.sh requirements: host: - jpeg diff --git a/tests/test_configure_feedstock.py b/tests/test_configure_feedstock.py index 8fef3d1f6..217671e37 100644 --- a/tests/test_configure_feedstock.py +++ b/tests/test_configure_feedstock.py @@ -5,6 +5,7 @@ import shutil import tempfile import textwrap +from itertools import product from pathlib import Path import pytest @@ -1981,6 +1982,47 @@ def test_get_used_key_values_by_input_order( assert used_key_values == expected_used_key_values +def test_reduce_variants(config_yaml, jinja_env): + if config_yaml.type == "rattler-build": + pytest.skip("only conda-build") + _thisdir = os.path.abspath(os.path.dirname(__file__)) + recipe = os.path.join(_thisdir, "recipes", "multiple_outputs") + dest_recipe = os.path.join(config_yaml.workdir, "recipe") + shutil.copytree(recipe, dest_recipe, dirs_exist_ok=True) + + forge_config = configure_feedstock._load_forge_config( + config_yaml.workdir, + exclusive_config_file=os.path.join( + config_yaml.workdir, "recipe", "default_config.yaml" + ), + ) + + configure_feedstock.render_azure( + jinja_env=jinja_env, + forge_config=forge_config, + forge_dir=config_yaml.workdir, + ) + ci_support = Path(config_yaml.workdir) / ".ci_support" + all_yamls = list(ci_support.glob("linux*.yaml")) + print(all_yamls) + matrix_entry = next(iter(ci_support.glob("linux*.yaml"))) + with matrix_entry.open() as f: + cbc = yaml.safe_load(f) + assert len(all_yamls) == 8 # 2 libpq * 2 libpng * 2 script_only + # top-level entries + assert "libpq" in cbc + assert len(cbc["libpq"]) == 1 + assert "libpng" in cbc + assert len(cbc["libpng"]) == 1 + assert "script_only" in cbc + assert len(cbc["script_only"]) == 1 + + # inner per-build variants + assert "jpeg" in cbc + assert len(cbc["jpeg"]) == 2 + assert "script_only_2" in cbc + assert len(cbc["script_only_2"]) == 2 + def test_conda_build_api_render_for_smithy(testing_workdir): import conda_build.api @@ -1988,12 +2030,19 @@ def test_conda_build_api_render_for_smithy(testing_workdir): recipe = os.path.join(_thisdir, "recipes", "multiple_outputs") dest_recipe = os.path.join(testing_workdir, "recipe") shutil.copytree(recipe, dest_recipe) - all_top_level_builds = { - ("1.5", "9.5"), - ("1.5", "9.6"), - ("1.6", "9.5"), - ("1.6", "9.6"), - } + with open(os.path.join(dest_recipe, "conda_build_config.yaml")) as f: + cbc = yaml.safe_load(f) + + # all variants + # libpng, libpq contribute to top-level build matrix entries + # jpeg, script_only, script_only_2 contribute to output-only variants (not top-level builds) + # make sure all variants are still captured + variant_keys = ("libpng", "libpq", "jpeg", "script_only", "script_only_2") + + all_top_level_builds = set( + tuple(map(str, variant)) + for variant in product(*(cbc[key] for key in variant_keys)) + ) cs_metas = configure_feedstock._conda_build_api_render_for_smithy( dest_recipe, @@ -2014,18 +2063,10 @@ def test_conda_build_api_render_for_smithy(testing_workdir): for meta, _, _ in cs_metas: for variant in meta.config.variants: top_level_builds.add( - ( - variant.get("libpng"), - variant.get("libpq"), - ) + tuple(variant.get(key) for key in variant_keys) ) variant = meta.config.variant - top_level_builds.add( - ( - variant.get("libpng"), - variant.get("libpq"), - ) - ) + top_level_builds.add(tuple(variant.get(key) for key in variant_keys)) assert len(top_level_builds) == len(all_top_level_builds) assert top_level_builds == all_top_level_builds cb_metas = conda_build.api.render( @@ -2048,18 +2089,10 @@ def test_conda_build_api_render_for_smithy(testing_workdir): for meta, _, _ in cb_metas: for variant in meta.config.variants: top_level_builds.add( - ( - variant.get("libpng"), - variant.get("libpq"), - ) + tuple(variant.get(key) for key in variant_keys) ) variant = meta.config.variant - top_level_builds.add( - ( - variant.get("libpng"), - variant.get("libpq"), - ) - ) + top_level_builds.add(tuple(variant.get(key) for key in variant_keys)) assert len(top_level_builds) < len(all_top_level_builds) assert top_level_builds.issubset(all_top_level_builds) From 471104cc8f954d5f1d93e821ad4b35d09e76af9f Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 18 Sep 2024 17:38:55 +0200 Subject: [PATCH 8/8] update multi-output matrix test to cover script vars --- .../multiple_outputs/build-output1.bat | 2 + .../recipes/multiple_outputs/build-output1.sh | 2 +- .../multiple_outputs/build-output2.bat | 2 + .../recipes/multiple_outputs/build-output2.sh | 2 +- .../multiple_outputs/conda_build_config.yaml | 10 ++- tests/recipes/multiple_outputs/meta.yaml | 6 +- tests/test_cli.py | 37 +++++++- tests/test_configure_feedstock.py | 85 ++++++------------- 8 files changed, 77 insertions(+), 69 deletions(-) create mode 100644 tests/recipes/multiple_outputs/build-output1.bat create mode 100644 tests/recipes/multiple_outputs/build-output2.bat diff --git a/tests/recipes/multiple_outputs/build-output1.bat b/tests/recipes/multiple_outputs/build-output1.bat new file mode 100644 index 000000000..2c05241aa --- /dev/null +++ b/tests/recipes/multiple_outputs/build-output1.bat @@ -0,0 +1,2 @@ +echo %script_all% +echo %script_bat% diff --git a/tests/recipes/multiple_outputs/build-output1.sh b/tests/recipes/multiple_outputs/build-output1.sh index 4a2464bfd..7bf834200 100644 --- a/tests/recipes/multiple_outputs/build-output1.sh +++ b/tests/recipes/multiple_outputs/build-output1.sh @@ -1 +1 @@ -echo "${script_only}" +echo "${script_all}" diff --git a/tests/recipes/multiple_outputs/build-output2.bat b/tests/recipes/multiple_outputs/build-output2.bat new file mode 100644 index 000000000..2c05241aa --- /dev/null +++ b/tests/recipes/multiple_outputs/build-output2.bat @@ -0,0 +1,2 @@ +echo %script_all% +echo %script_bat% diff --git a/tests/recipes/multiple_outputs/build-output2.sh b/tests/recipes/multiple_outputs/build-output2.sh index dfece6418..01c6bde97 100644 --- a/tests/recipes/multiple_outputs/build-output2.sh +++ b/tests/recipes/multiple_outputs/build-output2.sh @@ -1,2 +1,2 @@ -echo "${script_only}" +echo "${script_all}" echo "${script_only_2}" diff --git a/tests/recipes/multiple_outputs/conda_build_config.yaml b/tests/recipes/multiple_outputs/conda_build_config.yaml index f13fd1550..348b98773 100644 --- a/tests/recipes/multiple_outputs/conda_build_config.yaml +++ b/tests/recipes/multiple_outputs/conda_build_config.yaml @@ -29,13 +29,17 @@ unused: - a - b # used in all output scripts -script_only: +# single value +script_all: - a - - b -# used in output2 script +# used in output2 sh script script_only_2: - a2 - b2 +# used in both outputs, only windows +script_bat: + - a2 + - b2 # zip these just for testing purposes zip_keys: - diff --git a/tests/recipes/multiple_outputs/meta.yaml b/tests/recipes/multiple_outputs/meta.yaml index 875af14e2..5e7853fb0 100644 --- a/tests/recipes/multiple_outputs/meta.yaml +++ b/tests/recipes/multiple_outputs/meta.yaml @@ -14,7 +14,8 @@ requirements: outputs: - name: test_output_1 - script: build-output1.sh + script: build-output1.sh # [not win] + script: build-output1.bat # [win] requirements: - bzip2 test: @@ -22,7 +23,8 @@ outputs: - test -e $PREFIX/lib/libz.so # [linux] - test -e $PREFIX/lib/libz.dylib # [osx] - name: test_output_2 - script: build-output2.sh + script: build-output2.sh # [not win] + script: build-output2.bat # [win] requirements: host: - jpeg diff --git a/tests/test_cli.py b/tests/test_cli.py index e51027a4d..212eb56cb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,6 +3,7 @@ import os import shutil import subprocess +from pathlib import Path from textwrap import dedent import pytest @@ -114,19 +115,23 @@ def test_init_multiple_output_matrix(testing_workdir): temporary_directory=os.path.join(recipe, "temp"), ) regen_obj(args) - matrix_dir = os.path.join(feedstock_dir, ".ci_support") # the matrix should be consolidated among all outputs, as well as the top-level # reqs. Only the top-level reqs should have indedependent config files, # though - loops within outputs are contained in those top-level configs. - matrix_dir_len = len(os.listdir(matrix_dir)) - assert matrix_dir_len == 13 + matrix_dir = Path(feedstock_dir) / ".ci_support" + matrix_files = list(matrix_dir.glob("*.yaml")) + linux_libpng16 = matrix_dir / "linux_64_libpng1.6libpq9.5.yaml" + assert linux_libpng16 in matrix_files + assert len(matrix_files) == 16 linux_libpng16 = os.path.join( matrix_dir, "linux_64_libpng1.6libpq9.5.yaml" ) assert os.path.isfile(linux_libpng16) with open(linux_libpng16) as f: config = yaml.safe_load(f) + assert "libpng" in config assert config["libpng"] == ["1.6"] + assert "libpq" in config assert config["libpq"] == ["9.5"] # this is a zipped key, but it's not used, so it shouldn't show up assert "libtiff" not in config @@ -139,6 +144,32 @@ def test_init_multiple_output_matrix(testing_workdir): # not show up in the final configs. assert "zlib" not in config + # script vars are present + assert "script_all" in config + assert config["script_all"] == ["a"] + assert "script_only_2" in config + assert config["script_only_2"] == ["a2", "b2"] + + # windows script_bat shouldn't be in config + assert "script_bat" not in config + + # check windows, which has additional axis + win_config = matrix_dir / "win_64_libpng1.5libpq9.6script_bata2.yaml" + assert win_config in matrix_files + with win_config.open() as f: + config = yaml.safe_load(f) + + # script vars are present + assert "script_all" in config + assert config["script_all"] == ["a"] + + # sh-only var not in windows config + assert "script_only_2" not in config + + # windows script_bat should be in config + assert "script_bat" in config + assert config["script_bat"] == ["a2"] + @pytest.mark.parametrize( "dirname", ["multiple_outputs", "multiple_outputs2", "multiple_outputs3"] diff --git a/tests/test_configure_feedstock.py b/tests/test_configure_feedstock.py index 217671e37..8fef3d1f6 100644 --- a/tests/test_configure_feedstock.py +++ b/tests/test_configure_feedstock.py @@ -5,7 +5,6 @@ import shutil import tempfile import textwrap -from itertools import product from pathlib import Path import pytest @@ -1982,47 +1981,6 @@ def test_get_used_key_values_by_input_order( assert used_key_values == expected_used_key_values -def test_reduce_variants(config_yaml, jinja_env): - if config_yaml.type == "rattler-build": - pytest.skip("only conda-build") - _thisdir = os.path.abspath(os.path.dirname(__file__)) - recipe = os.path.join(_thisdir, "recipes", "multiple_outputs") - dest_recipe = os.path.join(config_yaml.workdir, "recipe") - shutil.copytree(recipe, dest_recipe, dirs_exist_ok=True) - - forge_config = configure_feedstock._load_forge_config( - config_yaml.workdir, - exclusive_config_file=os.path.join( - config_yaml.workdir, "recipe", "default_config.yaml" - ), - ) - - configure_feedstock.render_azure( - jinja_env=jinja_env, - forge_config=forge_config, - forge_dir=config_yaml.workdir, - ) - ci_support = Path(config_yaml.workdir) / ".ci_support" - all_yamls = list(ci_support.glob("linux*.yaml")) - print(all_yamls) - matrix_entry = next(iter(ci_support.glob("linux*.yaml"))) - with matrix_entry.open() as f: - cbc = yaml.safe_load(f) - assert len(all_yamls) == 8 # 2 libpq * 2 libpng * 2 script_only - # top-level entries - assert "libpq" in cbc - assert len(cbc["libpq"]) == 1 - assert "libpng" in cbc - assert len(cbc["libpng"]) == 1 - assert "script_only" in cbc - assert len(cbc["script_only"]) == 1 - - # inner per-build variants - assert "jpeg" in cbc - assert len(cbc["jpeg"]) == 2 - assert "script_only_2" in cbc - assert len(cbc["script_only_2"]) == 2 - def test_conda_build_api_render_for_smithy(testing_workdir): import conda_build.api @@ -2030,19 +1988,12 @@ def test_conda_build_api_render_for_smithy(testing_workdir): recipe = os.path.join(_thisdir, "recipes", "multiple_outputs") dest_recipe = os.path.join(testing_workdir, "recipe") shutil.copytree(recipe, dest_recipe) - with open(os.path.join(dest_recipe, "conda_build_config.yaml")) as f: - cbc = yaml.safe_load(f) - - # all variants - # libpng, libpq contribute to top-level build matrix entries - # jpeg, script_only, script_only_2 contribute to output-only variants (not top-level builds) - # make sure all variants are still captured - variant_keys = ("libpng", "libpq", "jpeg", "script_only", "script_only_2") - - all_top_level_builds = set( - tuple(map(str, variant)) - for variant in product(*(cbc[key] for key in variant_keys)) - ) + all_top_level_builds = { + ("1.5", "9.5"), + ("1.5", "9.6"), + ("1.6", "9.5"), + ("1.6", "9.6"), + } cs_metas = configure_feedstock._conda_build_api_render_for_smithy( dest_recipe, @@ -2063,10 +2014,18 @@ def test_conda_build_api_render_for_smithy(testing_workdir): for meta, _, _ in cs_metas: for variant in meta.config.variants: top_level_builds.add( - tuple(variant.get(key) for key in variant_keys) + ( + variant.get("libpng"), + variant.get("libpq"), + ) ) variant = meta.config.variant - top_level_builds.add(tuple(variant.get(key) for key in variant_keys)) + top_level_builds.add( + ( + variant.get("libpng"), + variant.get("libpq"), + ) + ) assert len(top_level_builds) == len(all_top_level_builds) assert top_level_builds == all_top_level_builds cb_metas = conda_build.api.render( @@ -2089,10 +2048,18 @@ def test_conda_build_api_render_for_smithy(testing_workdir): for meta, _, _ in cb_metas: for variant in meta.config.variants: top_level_builds.add( - tuple(variant.get(key) for key in variant_keys) + ( + variant.get("libpng"), + variant.get("libpq"), + ) ) variant = meta.config.variant - top_level_builds.add(tuple(variant.get(key) for key in variant_keys)) + top_level_builds.add( + ( + variant.get("libpng"), + variant.get("libpq"), + ) + ) assert len(top_level_builds) < len(all_top_level_builds) assert top_level_builds.issubset(all_top_level_builds)