diff --git a/docs/changes.rst b/docs/changes.rst index 4e23840e9..c3d2814f8 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -108,7 +108,7 @@ Release Notes --- * big changes in ``ShellTask``, ``DockerTask`` and ``SingularityTask`` - * customized input specification and output specification for ``Task``\s + * customized input definition and output definition for ``Task``\s * adding singularity checks to Travis CI * binding all input files to the container * changes in ``Workflow`` diff --git a/docs/components.rst b/docs/components.rst index d4928e82c..46dcacbe3 100644 --- a/docs/components.rst +++ b/docs/components.rst @@ -66,7 +66,7 @@ Shell Command Tasks The *Task* can accommodate more complex shell commands by allowing the user to customize inputs and outputs of the commands. One can generate an input - specification to specify names of inputs, positions in the command, types of + definition to specify names of inputs, positions in the command, types of the inputs, and other metadata. As a specific example, FSL's BET command (Brain Extraction Tool) can be called on the command line as: @@ -76,7 +76,7 @@ Shell Command Tasks bet input_file output_file -m Each of the command argument can be treated as a named input to the - ``ShellCommandTask``, and can be included in the input specification. + ``ShellCommandTask``, and can be included in the input definition. As shown next, even an output is specified by constructing the *out_file* field form a template: @@ -97,7 +97,7 @@ Shell Command Tasks ( "mask", bool, { "help_string": "create binary mask", "argstr": "-m", } ) ], - bases=(ShellSpec,) ) + bases=(ShellDef,) ) ShellCommandTask(executable="bet", input_spec=bet_input_spec) diff --git a/docs/input_spec.rst b/docs/input_spec.rst index 2940c1782..18679d5de 100644 --- a/docs/input_spec.rst +++ b/docs/input_spec.rst @@ -5,7 +5,7 @@ Input Specification As it was mentioned in :ref:`shell_command_task`, the user can customize the input and output for the `ShellCommandTask`. -In this section, more examples of the input specification will be provided. +In this section, more examples of the input definition will be provided. Let's start from the previous example: @@ -27,29 +27,29 @@ Let's start from the previous example: ( "mask", bool, { "help_string": "create binary mask", "argstr": "-m", } ) ], - bases=(ShellSpec,) ) + bases=(ShellDef,) ) ShellCommandTask(executable="bet", input_spec=bet_input_spec) -In order to create an input specification, a new `SpecInfo` object has to be created. +In order to create an input definition, a new `SpecInfo` object has to be created. The field `name` specifies the type of the spec and it should be always "Input" for -the input specification. -The field `bases` specifies the "base specification" you want to use (can think about it as a -`parent class`) and it will usually contains `ShellSpec` only, unless you want to build on top of -your other specification (this will not be cover in this section). +the input definition. +The field `bases` specifies the "base definition" you want to use (can think about it as a +`parent class`) and it will usually contains `ShellDef` only, unless you want to build on top of +your other definition (this will not be cover in this section). The part that should be always customised is the `fields` part. -Each element of the `fields` is a separate input field that is added to the specification. +Each element of the `fields` is a separate input field that is added to the definition. In this example, three-elements tuples - with name, type and dictionary with additional information - are used. But this is only one of the supported syntax, more options will be described below. -Adding a New Field to the Spec +Adding a New Field to the Def ------------------------------ -Pydra uses `attr` classes to represent the input specification, and the full syntax for each field +Pydra uses `attr` classes to represent the input definition, and the full syntax for each field is: .. code-block:: python @@ -152,15 +152,15 @@ In the example we used multiple keys in the metadata dictionary including `help_ `output_file_template` (`str`): If provided, the field is treated also as an output field and it is added to the output spec. The template can use other fields, e.g. `{file1}`. - Used in order to create an output specification. + Used in order to create an output definition. `output_field_name` (`str`, used together with `output_file_template`) If provided the field is added to the output spec with changed name. - Used in order to create an output specification. + Used in order to create an output definition. `keep_extension` (`bool`, default: `True`): A flag that specifies if the file extension should be removed from the field value. - Used in order to create an output specification. + Used in order to create an output definition. `readonly` (`bool`, default: `False`): If `True` the input field can't be provided by the user but it aggregates other input fields diff --git a/docs/output_spec.rst b/docs/output_spec.rst index 2e0907076..183e27333 100644 --- a/docs/output_spec.rst +++ b/docs/output_spec.rst @@ -5,7 +5,7 @@ Output Specification As it was mentioned in :ref:`shell_command_task`, the user can customize the input and output for the `ShellCommandTask`. -In this section, the output specification will be covered. +In this section, the output definition will be covered. Instead of using field with `output_file_template` in the customized `input_spec` to specify an output field, @@ -29,7 +29,7 @@ a customized `output_spec` can be used, e.g.: ), ) ], - bases=(ShellOutSpec,), + bases=(ShellOutDef,), ) ShellCommandTask(executable=executable, @@ -37,18 +37,18 @@ a customized `output_spec` can be used, e.g.: -Similarly as for `input_spec`, in order to create an output specification, +Similarly as for `input_spec`, in order to create an output definition, a new `SpecInfo` object has to be created. The field `name` specifies the type of the spec and it should be always "Output" for -the output specification. -The field `bases` specifies the "base specification" you want to use (can think about it as a -`parent class`) and it will usually contains `ShellOutSpec` only, unless you want to build on top of -your other specification (this will not be cover in this section). +the output definition. +The field `bases` specifies the "base definition" you want to use (can think about it as a +`parent class`) and it will usually contains `ShellOutDef` only, unless you want to build on top of +your other definition (this will not be cover in this section). The part that should be always customised is the `fields` part. -Each element of the `fields` is a separate output field that is added to the specification. +Each element of the `fields` is a separate output field that is added to the definition. In this example, a three-elements tuple - with name, type and dictionary with additional information - is used. -See :ref:`Input Specification section` for other recognized syntax for specification's fields +See :ref:`Input Specification section` for other recognized syntax for definition's fields and possible types. diff --git a/new-docs/source/index.rst b/new-docs/source/index.rst index e95ea081b..ca804f083 100644 --- a/new-docs/source/index.rst +++ b/new-docs/source/index.rst @@ -3,11 +3,21 @@ Pydra ===== -Pydra is a new lightweight dataflow engine written in Python, which provides a simple way to -implement scientific workflows that use a mix of shell commands and Python functions. - -Pydra is developed as an open-source project in the neuroimaging community, -but it is designed as a general-purpose dataflow engine to support any scientific domain. +Pydra is a lightweight, Python 3.11+ dataflow engine for computational graph construction, +manipulation, and distributed execution. Designed as a successor to created for [Nipype](https://github.com/nipy/nipype), +Pydra is a general-purpose engine that supports analytics in any scientific domain. +Pydra helps build reproducible, scalable, reusable, and fully automated, provenance +tracked scientific workflows that combine Python functions and shell commands. + +The power of Pydra lies in ease of workflow creation and execution for complex +multiparameter map-reduce operations, and the use of global cache. + +Pydra's key features are: +- Modular execution backends (see [Advanced execution](../tutorial/advanced-execution.html)) +- Map-reduce like semantics (see [Splitting and combining](../explanation/splitting-combining.html)) +- Global cache support to reduce recomputation (see [Hashing and caching](../explanation/hashing-caching.html)) +- Support for execution of Tasks in containerized environments (see [Environments](../explanation/environments.html)) +- Strong type-checking and type-hinting support (see [Typing](../explanation/typing.html)) See :ref:`Design philosophy` for more an explanation of the design of Pydra. @@ -64,7 +74,7 @@ Indices and tables .. toctree:: :maxdepth: 2 - :caption: Execution + :caption: Task execution :hidden: tutorial/getting-started diff --git a/new-docs/source/tutorial/advanced-execution.ipynb b/new-docs/source/tutorial/advanced-execution.ipynb index 1105a8f31..63a7673da 100644 --- a/new-docs/source/tutorial/advanced-execution.ipynb +++ b/new-docs/source/tutorial/advanced-execution.ipynb @@ -11,7 +11,14 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Plugins" + "## Workers\n", + "\n", + "Pydra supports several workers with which to execute tasks\n", + "\n", + "- `ConcurrentFutures`\n", + "- `SLURM`\n", + "- `Dask` (experimental)\n", + "- `Serial` (for debugging)" ] }, { diff --git a/new-docs/source/tutorial/python.ipynb b/new-docs/source/tutorial/python.ipynb index d251a93da..a80458a06 100644 --- a/new-docs/source/tutorial/python.ipynb +++ b/new-docs/source/tutorial/python.ipynb @@ -1,299 +1,299 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Python-tasks" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "funcOutputs(out=2.0)\n" - ] - } - ], - "source": [ - "from pydra.design import python\n", - "\n", - "\n", - "def func(a: int) -> float:\n", - " \"\"\"Sample function with inputs and outputs\"\"\"\n", - " return a * 2\n", - "\n", - "SampleSpec = python.define(func)\n", - "\n", - "spec = SampleSpec(a=1)\n", - "result = spec()\n", - "print(result.output)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### With typing" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "def func(a: int, k: float = 2.0) -> float:\n", - " \"\"\"Sample function with inputs and outputs\"\"\"\n", - " return a * k\n", - "\n", - "SampleSpec = python.define(func)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Augment with explicit inputs and outputs\n" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "from decimal import Decimal\n", - "\n", - "def func(a: int) -> float:\n", - " \"\"\"Sample function with inputs and outputs\"\"\"\n", - " return a * 2\n", - "\n", - "SampleSpec = python.define(\n", - " func,\n", - " inputs={\"a\": python.arg(help_string=\"The argument to be doubled\")},\n", - " outputs={\"b\": python.out(help_string=\"the doubled output\", type=Decimal)},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Decorated_function" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "# Note we use CamelCase as the function is translated to a class\n", - "\n", - "@python.define(outputs=[\"c\", \"d\"])\n", - "def SampleSpec(a: int, b: float) -> tuple[float, float]:\n", - " \"\"\"Sample function for testing\"\"\"\n", - " return a + b, a * b" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "## Pull helps from docstring" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'a': arg(name='a', type=, default=EMPTY, help_string='First input to be inputted', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False),\n", - " 'b': arg(name='b', type=, default=EMPTY, help_string='Second input', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False),\n", - " 'function': arg(name='function', type=typing.Callable, default=, help_string='', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False)}\n", - "{'c': out(name='c', type=, default=EMPTY, help_string='Sum of a and b', requires=[], converter=None, validator=None),\n", - " 'd': out(name='d', type=, default=EMPTY, help_string='Product of a and b', requires=[], converter=None, validator=None)}\n" - ] - } - ], - "source": [ - "from pprint import pprint\n", - "from pydra.engine.helpers import fields_dict\n", - "\n", - "@python.define(outputs=[\"c\", \"d\"])\n", - "def SampleSpec(a: int, b: float) -> tuple[float, float]:\n", - " \"\"\"Sample function for testing\n", - "\n", - " Args:\n", - " a: First input\n", - " to be inputted\n", - " b: Second input\n", - "\n", - " Returns:\n", - " c: Sum of a and b\n", - " d: Product of a and b\n", - " \"\"\"\n", - " return a + b, a * b\n", - "\n", - "pprint(fields_dict(SampleSpec))\n", - "pprint(fields_dict(SampleSpec.Outputs))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Dataclass form" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'a': arg(name='a', type=, default=EMPTY, help_string='First input to be inputted', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False),\n", - " 'b': arg(name='b', type=, default=2.0, help_string='Second input', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False),\n", - " 'function': arg(name='function', type=typing.Callable, default=, help_string='', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False)}\n", - "{'c': out(name='c', type=, default=EMPTY, help_string='Sum of a and b', requires=[], converter=None, validator=None),\n", - " 'd': out(name='d', type=, default=EMPTY, help_string='Product of a and b', requires=[], converter=None, validator=None)}\n" - ] - } - ], - "source": [ - "\n", - "@python.define\n", - "class SampleSpec:\n", - " \"\"\"Sample class for testing\n", - "\n", - " Args:\n", - " a: First input\n", - " to be inputted\n", - " b: Second input\n", - " \"\"\"\n", - "\n", - " a: int\n", - " b: float = 2.0\n", - "\n", - " class Outputs:\n", - " \"\"\"\n", - " Args:\n", - " c: Sum of a and b\n", - " d: Product of a and b\n", - " \"\"\"\n", - "\n", - " c: float\n", - " d: float\n", - "\n", - " @staticmethod\n", - " def function(a, b):\n", - " return a + b, a * b\n", - "\n", - "pprint(fields_dict(SampleSpec))\n", - "pprint(fields_dict(SampleSpec.Outputs))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Canonical form (to work with static type-checking)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'a': arg(name='a', type=, default=EMPTY, help_string='First input to be inputted', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False),\n", - " 'b': arg(name='b', type=, default=EMPTY, help_string='Second input', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False),\n", - " 'function': arg(name='function', type=typing.Callable, default=, help_string='', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False)}\n", - "{'c': out(name='c', type=, default=EMPTY, help_string='Sum of a and b', requires=[], converter=None, validator=None),\n", - " 'd': out(name='d', type=, default=EMPTY, help_string='Product of a and b', requires=[], converter=None, validator=None)}\n" - ] - } - ], - "source": [ - "from pydra.engine.specs import PythonSpec, PythonOutputs\n", - "\n", - "@python.define\n", - "class SampleSpec(PythonSpec[\"SampleSpec.Outputs\"]):\n", - " \"\"\"Sample class for testing\n", - "\n", - " Args:\n", - " a: First input\n", - " to be inputted\n", - " b: Second input\n", - " \"\"\"\n", - "\n", - " a: int\n", - " b: float\n", - "\n", - " @python.outputs\n", - " class Outputs(PythonOutputs):\n", - " \"\"\"\n", - " Args:\n", - " c: Sum of a and b\n", - " d: Product of a and b\n", - " \"\"\"\n", - "\n", - " c: float\n", - " d: float\n", - "\n", - " @staticmethod\n", - " def function(a, b):\n", - " return a + b, a * b\n", - "\n", - "pprint(fields_dict(SampleSpec))\n", - "pprint(fields_dict(SampleSpec.Outputs))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "wf12", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.5" - } - }, - "nbformat": 4, - "nbformat_minor": 2 + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Python-tasks" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "funcOutputs(out=2.0)\n" + ] + } + ], + "source": [ + "from pydra.design import python\n", + "\n", + "\n", + "def func(a: int) -> float:\n", + " \"\"\"Sample function with inputs and outputs\"\"\"\n", + " return a * 2\n", + "\n", + "SampleDef = python.define(func)\n", + "\n", + "spec = SampleDef(a=1)\n", + "result = spec()\n", + "print(result.output)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### With typing" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "def func(a: int, k: float = 2.0) -> float:\n", + " \"\"\"Sample function with inputs and outputs\"\"\"\n", + " return a * k\n", + "\n", + "SampleDef = python.define(func)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Augment with explicit inputs and outputs\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "from decimal import Decimal\n", + "\n", + "def func(a: int) -> float:\n", + " \"\"\"Sample function with inputs and outputs\"\"\"\n", + " return a * 2\n", + "\n", + "SampleDef = python.define(\n", + " func,\n", + " inputs={\"a\": python.arg(help_string=\"The argument to be doubled\")},\n", + " outputs={\"b\": python.out(help_string=\"the doubled output\", type=Decimal)},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Decorated_function" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "# Note we use CamelCase as the function is translated to a class\n", + "\n", + "@python.define(outputs=[\"c\", \"d\"])\n", + "def SampleDef(a: int, b: float) -> tuple[float, float]:\n", + " \"\"\"Sample function for testing\"\"\"\n", + " return a + b, a * b" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Pull helps from docstring" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'a': arg(name='a', type=, default=EMPTY, help_string='First input to be inputted', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False),\n", + " 'b': arg(name='b', type=, default=EMPTY, help_string='Second input', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False),\n", + " 'function': arg(name='function', type=typing.Callable, default=, help_string='', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False)}\n", + "{'c': out(name='c', type=, default=EMPTY, help_string='Sum of a and b', requires=[], converter=None, validator=None),\n", + " 'd': out(name='d', type=, default=EMPTY, help_string='Product of a and b', requires=[], converter=None, validator=None)}\n" + ] + } + ], + "source": [ + "from pprint import pprint\n", + "from pydra.engine.helpers import fields_dict\n", + "\n", + "@python.define(outputs=[\"c\", \"d\"])\n", + "def SampleDef(a: int, b: float) -> tuple[float, float]:\n", + " \"\"\"Sample function for testing\n", + "\n", + " Args:\n", + " a: First input\n", + " to be inputted\n", + " b: Second input\n", + "\n", + " Returns:\n", + " c: Sum of a and b\n", + " d: Product of a and b\n", + " \"\"\"\n", + " return a + b, a * b\n", + "\n", + "pprint(fields_dict(SampleDef))\n", + "pprint(fields_dict(SampleDef.Outputs))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Dataclass form" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'a': arg(name='a', type=, default=EMPTY, help_string='First input to be inputted', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False),\n", + " 'b': arg(name='b', type=, default=2.0, help_string='Second input', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False),\n", + " 'function': arg(name='function', type=typing.Callable, default=, help_string='', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False)}\n", + "{'c': out(name='c', type=, default=EMPTY, help_string='Sum of a and b', requires=[], converter=None, validator=None),\n", + " 'd': out(name='d', type=, default=EMPTY, help_string='Product of a and b', requires=[], converter=None, validator=None)}\n" + ] + } + ], + "source": [ + "\n", + "@python.define\n", + "class SampleDef:\n", + " \"\"\"Sample class for testing\n", + "\n", + " Args:\n", + " a: First input\n", + " to be inputted\n", + " b: Second input\n", + " \"\"\"\n", + "\n", + " a: int\n", + " b: float = 2.0\n", + "\n", + " class Outputs:\n", + " \"\"\"\n", + " Args:\n", + " c: Sum of a and b\n", + " d: Product of a and b\n", + " \"\"\"\n", + "\n", + " c: float\n", + " d: float\n", + "\n", + " @staticmethod\n", + " def function(a, b):\n", + " return a + b, a * b\n", + "\n", + "pprint(fields_dict(SampleDef))\n", + "pprint(fields_dict(SampleDef.Outputs))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Canonical form (to work with static type-checking)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'a': arg(name='a', type=, default=EMPTY, help_string='First input to be inputted', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False),\n", + " 'b': arg(name='b', type=, default=EMPTY, help_string='Second input', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False),\n", + " 'function': arg(name='function', type=typing.Callable, default=, help_string='', requires=[], converter=None, validator=None, allowed_values=(), xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False)}\n", + "{'c': out(name='c', type=, default=EMPTY, help_string='Sum of a and b', requires=[], converter=None, validator=None),\n", + " 'd': out(name='d', type=, default=EMPTY, help_string='Product of a and b', requires=[], converter=None, validator=None)}\n" + ] + } + ], + "source": [ + "from pydra.engine.specs import PythonDef, PythonOutputs\n", + "\n", + "@python.define\n", + "class SampleDef(PythonDef[\"SampleDef.Outputs\"]):\n", + " \"\"\"Sample class for testing\n", + "\n", + " Args:\n", + " a: First input\n", + " to be inputted\n", + " b: Second input\n", + " \"\"\"\n", + "\n", + " a: int\n", + " b: float\n", + "\n", + " @python.outputs\n", + " class Outputs(PythonOutputs):\n", + " \"\"\"\n", + " Args:\n", + " c: Sum of a and b\n", + " d: Product of a and b\n", + " \"\"\"\n", + "\n", + " c: float\n", + " d: float\n", + "\n", + " @staticmethod\n", + " def function(a, b):\n", + " return a + b, a * b\n", + "\n", + "pprint(fields_dict(SampleDef))\n", + "pprint(fields_dict(SampleDef.Outputs))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "wf12", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/new-docs/source/tutorial/shell.ipynb b/new-docs/source/tutorial/shell.ipynb index 5416998b6..fa6849237 100644 --- a/new-docs/source/tutorial/shell.ipynb +++ b/new-docs/source/tutorial/shell.ipynb @@ -1,524 +1,524 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Shell-tasks" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Command-line templates\n", - "\n", - "Shell task specs can be defined using from string templates that resemble the command-line usage examples typically used in in-line help. Therefore, they can be quick and intuitive way to specify a shell task. For example, a simple spec for the copy command `cp` that omits optional flags," - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from pydra.design import shell\n", - "\n", - "Cp = shell.define(\"cp \")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Input and output fields are both specified by placing the name of the field within enclosing `<` and `>`. Outputs are differentiated by the `out|` prefix.\n", - "\n", - "This shell task can then be run just as a Python task would be run, first parameterising it, then executing" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Command-line to be run: cp /var/folders/mz/yn83q2fd3s758w1j75d2nnw80000gn/T/tmpoyx19gql/in.txt /var/folders/mz/yn83q2fd3s758w1j75d2nnw80000gn/T/tmpoyx19gql/out.txt\n", - "Contents of copied file ('/var/folders/mz/yn83q2fd3s758w1j75d2nnw80000gn/T/tmpoyx19gql/out.txt'): 'Contents to be copied'\n" - ] - } - ], - "source": [ - "from pathlib import Path\n", - "from tempfile import mkdtemp\n", - "\n", - "# Make a test file to copy\n", - "test_dir = Path(mkdtemp())\n", - "test_file = test_dir / \"in.txt\"\n", - "with open(test_file, \"w\") as f:\n", - " f.write(\"Contents to be copied\")\n", - "\n", - "# Parameterise the task spec\n", - "cp = Cp(in_file=test_file, destination=test_dir / \"out.txt\")\n", - "\n", - "# Print the cmdline to be run to double check\n", - "print(f\"Command-line to be run: {cp.cmdline}\")\n", - "\n", - "# Run the shell-comand task\n", - "result = cp()\n", - "\n", - "print(\n", - " f\"Contents of copied file ('{result.output.destination}'): \"\n", - " f\"'{Path(result.output.destination).read_text()}'\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If paths to output files are not provided in the parameterisation, it will default to the name of the field" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "cp /var/folders/mz/yn83q2fd3s758w1j75d2nnw80000gn/T/tmpoyx19gql/in.txt /Users/tclose/git/workflows/pydra/docs/source/tutorial/destination\n" - ] - } - ], - "source": [ - "cp = Cp(in_file=test_file)\n", - "print(cp.cmdline)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Specifying types\n", - "\n", - "By default, shell-command fields are considered to be of `fileformats.generic.FsObject` type. However, more specific file formats or built-in Python types can be specified by appending the type to the field name after a `:`.\n", - "\n", - "File formats are specified by their MIME type or \"MIME-like\" strings (see the [FileFormats docs](https://arcanaframework.github.io/fileformats/mime.html) for details)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trim-png /mock/png.png /path/to/output.png\n" - ] - } - ], - "source": [ - "from fileformats.image import Png\n", - "\n", - "TrimPng = shell.define(\"trim-png \")\n", - "\n", - "trim_png = TrimPng(in_image=Png.mock(), out_image=\"/path/to/output.png\")\n", - "\n", - "print(trim_png.cmdline)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Flags and options\n", - "\n", - "Command line flags can also be added to the shell template, either the single or double hyphen form. The field template name immediately following the flag will be associate with that flag.\n", - "\n", - "If there is no space between the flag and the field template, then the field is assumed to be a boolean, otherwise it is assumed to be of type string unless otherwise specified.\n", - "\n", - "If a field is optional, the field template should end with a `?`. Tuple fields are specified by comma separated types.\n", - "\n", - "Varargs are specified by the type followed by an ellipsis, e.g. ``" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'executable': arg(name='executable', type=typing.Union[str, typing.Sequence[str]], default='cp', help_string=\"the first part of the command, can be a string, e.g. 'ls', or a list, e.g. ['ls', '-l', 'dirname']\", requires=[], converter=None, validator=, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=0, sep=None, allowed_values=None, container_path=False, formatter=None),\n", - " 'in_fs_objects': arg(name='in_fs_objects', type=pydra.utils.typing.MultiInputObj[fileformats.generic.fsobject.FsObject], default=EMPTY, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=1, sep=' ', allowed_values=None, container_path=False, formatter=None),\n", - " 'int_arg': arg(name='int_arg', type=int | None, default=None, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='--int-arg', position=5, sep=None, allowed_values=None, container_path=False, formatter=None),\n", - " 'out_dir': outarg(name='out_dir', type=, default=EMPTY, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=2, sep=None, allowed_values=None, container_path=False, formatter=None, path_template='out_dir', keep_extension=False),\n", - " 'recursive': arg(name='recursive', type=, default=False, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='-R', position=3, sep=None, allowed_values=None, container_path=False, formatter=None),\n", - " 'text_arg': arg(name='text_arg', type=str | None, default=None, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='--text-arg', position=4, sep=None, allowed_values=None, container_path=False, formatter=None),\n", - " 'tuple_arg': arg(name='tuple_arg', type=tuple[int, str] | None, default=None, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='--tuple-arg', position=6, sep=None, allowed_values=None, container_path=False, formatter=None)}\n", - "{'out_dir': outarg(name='out_dir', type=, default=EMPTY, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=2, sep=None, allowed_values=None, container_path=False, formatter=None, path_template='out_dir', keep_extension=False),\n", - " 'return_code': out(name='return_code', type=, default=EMPTY, help_string=\"The process' exit code.\", requires=[], converter=None, validator=None, callable=None),\n", - " 'stderr': out(name='stderr', type=, default=EMPTY, help_string='The standard error stream produced by the command.', requires=[], converter=None, validator=None, callable=None),\n", - " 'stdout': out(name='stdout', type=, default=EMPTY, help_string='The standard output stream produced by the command.', requires=[], converter=None, validator=None, callable=None)}\n" - ] - } - ], - "source": [ - "from pprint import pprint\n", - "from pydra.engine.helpers import fields_dict\n", - "\n", - "Cp = shell.define(\n", - " (\n", - " \"cp \"\n", - " \"-R \"\n", - " \"--text-arg \"\n", - " \"--int-arg \"\n", - " \"--tuple-arg \"\n", - " ),\n", - " )\n", - "\n", - "pprint(fields_dict(Cp))\n", - "pprint(fields_dict(Cp.Outputs))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Defaults\n", - "\n", - "Defaults can be specified by appending them to the field template after `=`" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "'--int-arg' default: 99\n" - ] - } - ], - "source": [ - "Cp = shell.define(\n", - " (\n", - " \"cp \"\n", - " \"-R \"\n", - " \"--text-arg \"\n", - " \"--int-arg \"\n", - " \"--tuple-arg \"\n", - " ),\n", - " )\n", - "\n", - "print(f\"'--int-arg' default: {fields_dict(Cp)['int_arg'].default}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Additional field attributes\n", - "\n", - "Additional attributes of the fields in the template can be specified by providing `shell.arg` or `shell.outarg` fields to the `inputs` and `outputs` keyword arguments to the define" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'executable': arg(name='executable', type=typing.Union[str, typing.Sequence[str]], default='cp', help_string=\"the first part of the command, can be a string, e.g. 'ls', or a list, e.g. ['ls', '-l', 'dirname']\", requires=[], converter=None, validator=, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=0, sep=None, allowed_values=None, container_path=False, formatter=None),\n", - " 'in_fs_objects': arg(name='in_fs_objects', type=pydra.utils.typing.MultiInputObj[fileformats.generic.fsobject.FsObject], default=EMPTY, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=1, sep=' ', allowed_values=None, container_path=False, formatter=None),\n", - " 'int_arg': arg(name='int_arg', type=int | None, default=None, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='--int-arg', position=4, sep=None, allowed_values=None, container_path=False, formatter=None),\n", - " 'out_dir': outarg(name='out_dir', type=, default=EMPTY, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=-2, sep=None, allowed_values=None, container_path=False, formatter=None, path_template='out_dir', keep_extension=False),\n", - " 'out_file': outarg(name='out_file', type=fileformats.generic.file.File | None, default=None, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=-1, sep=None, allowed_values=None, container_path=False, formatter=None, path_template='out_file', keep_extension=False),\n", - " 'recursive': arg(name='recursive', type=, default=False, help_string='If source_file designates a directory, cp copies the directory and the entire subtree connected at that point.', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='-R', position=2, sep=None, allowed_values=None, container_path=False, formatter=None),\n", - " 'text_arg': arg(name='text_arg', type=, default=EMPTY, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='--text-arg', position=3, sep=None, allowed_values=None, container_path=False, formatter=None),\n", - " 'tuple_arg': arg(name='tuple_arg', type=tuple[int, str], default=EMPTY, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='--tuple-arg', position=5, sep=None, allowed_values=None, container_path=False, formatter=None)}\n", - "{'out_dir': outarg(name='out_dir', type=, default=EMPTY, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=-2, sep=None, allowed_values=None, container_path=False, formatter=None, path_template='out_dir', keep_extension=False),\n", - " 'out_file': outarg(name='out_file', type=fileformats.generic.file.File | None, default=None, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=-1, sep=None, allowed_values=None, container_path=False, formatter=None, path_template='out_file', keep_extension=False),\n", - " 'return_code': out(name='return_code', type=, default=EMPTY, help_string=\"The process' exit code.\", requires=[], converter=None, validator=None, callable=None),\n", - " 'stderr': out(name='stderr', type=, default=EMPTY, help_string='The standard error stream produced by the command.', requires=[], converter=None, validator=None, callable=None),\n", - " 'stdout': out(name='stdout', type=, default=EMPTY, help_string='The standard output stream produced by the command.', requires=[], converter=None, validator=None, callable=None)}\n" - ] - } - ], - "source": [ - "Cp = shell.define(\n", - " (\n", - " \"cp \"\n", - " \"-R \"\n", - " \"--text-arg \"\n", - " \"--int-arg \"\n", - " \"--tuple-arg \"\n", - " ),\n", - " inputs={\"recursive\": shell.arg(\n", - " help_string=(\n", - " \"If source_file designates a directory, cp copies the directory and \"\n", - " \"the entire subtree connected at that point.\"\n", - " )\n", - " )},\n", - " outputs={\n", - " \"out_dir\": shell.outarg(position=-2),\n", - " \"out_file\": shell.outarg(position=-1),\n", - " },\n", - " )\n", - "\n", - "\n", - "pprint(fields_dict(Cp))\n", - "pprint(fields_dict(Cp.Outputs))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Callable outptus\n", - "\n", - "In addition to outputs that are specified to the tool on the command line, outputs can be derived from the outputs of the tool by providing a Python function that can take the output directory and inputs as arguments and return the output value. Callables can be either specified in the `callable` attribute of the `shell.out` field, or in a dictionary mapping the output name to the callable" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Size of the output file is: 256\n" - ] - } - ], - "source": [ - "import os\n", - "from pydra.design import shell\n", - "from pathlib import Path\n", - "from fileformats.generic import File\n", - "\n", - "# Arguments to the callable function can be one of \n", - "def get_file_size(out_file: Path) -> int:\n", - " \"\"\"Calculate the file size\"\"\"\n", - " result = os.stat(out_file)\n", - " return result.st_size\n", - "\n", - "\n", - "CpWithSize = shell.define(\n", - " \"cp \",\n", - " outputs={\"out_file_size\": get_file_size},\n", - ")\n", - "\n", - "# Parameterise the task spec\n", - "cp_with_size = CpWithSize(in_file=File.sample())\n", - "\n", - "# Run the command\n", - "result = cp_with_size()\n", - "\n", - "\n", - "print(f\"Size of the output file is: {result.output.out_file_size}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The callable can take any combination of the following arguments, which will be passed\n", - "to it when it is called\n", - "\n", - "* field: the `Field` object to be provided a value, useful when writing generic callables\n", - "* output_dir: a `Path` object referencing the working directory the command was run within\n", - "* inputs: a dictionary containing all the resolved inputs to the task\n", - "* stdout: the standard output stream produced by the command\n", - "* stderr: the standard error stream produced by the command\n", - "* *name of an input*: the name of any of the input arguments to the task, including output args that are part of the command line (i.e. output files)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Dataclass form\n", - "\n", - "Like with Python tasks, shell-tasks can also be specified in dataclass-form by using `shell.define` as a decorator." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'executable': arg(name='executable', type=typing.Union[str, typing.Sequence[str]], default='cp', help_string=\"the first part of the command, can be a string, e.g. 'ls', or a list, e.g. ['ls', '-l', 'dirname']\", requires=[], converter=None, validator=, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=0, sep=None, allowed_values=None, container_path=False, formatter=None),\n", - " 'in_fs_objects': arg(name='in_fs_objects', type=pydra.utils.typing.MultiInputObj[fileformats.generic.fsobject.FsObject], default=EMPTY, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=5, sep=None, allowed_values=None, container_path=False, formatter=None),\n", - " 'int_arg': arg(name='int_arg', type=int | None, default=None, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='--int-arg', position=1, sep=None, allowed_values=None, container_path=False, formatter=None),\n", - " 'recursive': arg(name='recursive', type=, default=False, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='-R', position=2, sep=None, allowed_values=None, container_path=False, formatter=None),\n", - " 'text_arg': arg(name='text_arg', type=, default=EMPTY, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='--text-arg', position=3, sep=None, allowed_values=None, container_path=False, formatter=None),\n", - " 'tuple_arg': arg(name='tuple_arg', type=tuple[int, str] | None, default=None, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='--tuple-arg', position=4, sep=None, allowed_values=None, container_path=False, formatter=None)}\n", - "{'out_file': out(name='out_file', type=, default=EMPTY, help_string='', requires=[], converter=None, validator=None, callable=None),\n", - " 'out_file_size': out(name='out_file_size', type=, default=EMPTY, help_string='', requires=[], converter=None, validator=None, callable=),\n", - " 'return_code': out(name='return_code', type=, default=EMPTY, help_string=\"The process' exit code.\", requires=[], converter=None, validator=None, callable=None),\n", - " 'stderr': out(name='stderr', type=, default=EMPTY, help_string='The standard error stream produced by the command.', requires=[], converter=None, validator=None, callable=None),\n", - " 'stdout': out(name='stdout', type=, default=EMPTY, help_string='The standard output stream produced by the command.', requires=[], converter=None, validator=None, callable=None)}\n" - ] - } - ], - "source": [ - "from fileformats.generic import FsObject, Directory\n", - "from pydra.utils.typing import MultiInputObj\n", - "\n", - "@shell.define\n", - "class CpWithSize:\n", - "\n", - " executable = \"cp\"\n", - "\n", - " in_fs_objects: MultiInputObj[FsObject]\n", - " recursive: bool = shell.arg(argstr=\"-R\")\n", - " text_arg: str = shell.arg(argstr=\"--text-arg\")\n", - " int_arg: int | None = shell.arg(argstr=\"--int-arg\")\n", - " tuple_arg: tuple[int, str] | None = shell.arg(argstr=\"--tuple-arg\")\n", - "\n", - " class Outputs:\n", - " out_file: File\n", - " out_file_size: int = shell.out(callable=get_file_size)\n", - "\n", - "\n", - "pprint(fields_dict(CpWithSize))\n", - "pprint(fields_dict(CpWithSize.Outputs))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To make workflows that use the interface type-checkable, the canonical form of a shell\n", - "task dataclass should inherit from `shell.Spec` parameterized by its nested Outputs class,\n", - "and the `Outputs` nested class should inherit from `shell.Outputs`." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "from pydra.engine.specs import ShellSpec, ShellOutputs\n", - "\n", - "@shell.define\n", - "class Cp(ShellSpec[\"Cp.Outputs\"]):\n", - "\n", - " executable = \"cp\"\n", - "\n", - " in_fs_objects: MultiInputObj[FsObject]\n", - " recursive: bool = shell.arg(argstr=\"-R\", default=False)\n", - " text_arg: str = shell.arg(argstr=\"--text-arg\")\n", - " int_arg: int | None = shell.arg(argstr=\"--int-arg\")\n", - " tuple_arg: tuple[int, str] | None = shell.arg(argstr=\"--tuple-arg\")\n", - "\n", - " @shell.outputs\n", - " class Outputs(ShellOutputs):\n", - " out_dir: Directory = shell.outarg(path_template=\"{out_dir}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Dynamic specifications\n", - "\n", - "In some cases, it is required to generate the specification for a task dynamically, which can be done by just providing the executable to `shell.define` and specifying all inputs and outputs explicitly" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ACommand input fields: [arg(name='in_file', type=, default=EMPTY, help_string='output file', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=-2, sep=None, allowed_values=None, container_path=False, formatter=None), outarg(name='out_file', type=, default=EMPTY, help_string='output file', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=-1, sep=None, allowed_values=None, container_path=False, formatter=None, path_template=None, keep_extension=False), arg(name='executable', type=typing.Union[str, typing.Sequence[str]], default='a-command', help_string=\"the first part of the command, can be a string, e.g. 'ls', or a list, e.g. ['ls', '-l', 'dirname']\", requires=[], converter=None, validator=, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=0, sep=None, allowed_values=None, container_path=False, formatter=None)]\n", - "ACommand input fields: [outarg(name='out_file', type=, default=EMPTY, help_string='output file', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=-1, sep=None, allowed_values=None, container_path=False, formatter=None, path_template=None, keep_extension=False), out(name='out_file_size', type=, default=EMPTY, help_string='size of the output directory', requires=[], converter=None, validator=None, callable=), out(name='return_code', type=, default=EMPTY, help_string=\"The process' exit code.\", requires=[], converter=None, validator=None, callable=None), out(name='stdout', type=, default=EMPTY, help_string='The standard output stream produced by the command.', requires=[], converter=None, validator=None, callable=None), out(name='stderr', type=, default=EMPTY, help_string='The standard error stream produced by the command.', requires=[], converter=None, validator=None, callable=None)]\n" - ] - } - ], - "source": [ - "from fileformats.generic import File\n", - "from pydra.engine.helpers import list_fields\n", - "\n", - "ACommand = shell.define(\n", - " \"a-command\",\n", - " inputs={\n", - " \"in_file\": shell.arg(type=File, help_string=\"output file\", argstr=\"\", position=-2)\n", - " },\n", - " outputs={\n", - " \"out_file\": shell.outarg(\n", - " type=File, help_string=\"output file\", argstr=\"\", position=-1\n", - " ),\n", - " \"out_file_size\": {\n", - " \"type\": int,\n", - " \"help_string\": \"size of the output directory\",\n", - " \"callable\": get_file_size,\n", - " }\n", - " },\n", - ")\n", - "\n", - "\n", - "print(f\"ACommand input fields: {list_fields(ACommand)}\")\n", - "print(f\"ACommand input fields: {list_fields(ACommand.Outputs)}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "wf12", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.5" - } - }, - "nbformat": 4, - "nbformat_minor": 2 + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Shell-tasks" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Command-line templates\n", + "\n", + "Shell task specs can be defined using from string templates that resemble the command-line usage examples typically used in in-line help. Therefore, they can be quick and intuitive way to specify a shell task. For example, a simple spec for the copy command `cp` that omits optional flags," + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from pydra.design import shell\n", + "\n", + "Cp = shell.define(\"cp \")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Input and output fields are both specified by placing the name of the field within enclosing `<` and `>`. Outputs are differentiated by the `out|` prefix.\n", + "\n", + "This shell task can then be run just as a Python task would be run, first parameterising it, then executing" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Command-line to be run: cp /var/folders/mz/yn83q2fd3s758w1j75d2nnw80000gn/T/tmpoyx19gql/in.txt /var/folders/mz/yn83q2fd3s758w1j75d2nnw80000gn/T/tmpoyx19gql/out.txt\n", + "Contents of copied file ('/var/folders/mz/yn83q2fd3s758w1j75d2nnw80000gn/T/tmpoyx19gql/out.txt'): 'Contents to be copied'\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "from tempfile import mkdtemp\n", + "\n", + "# Make a test file to copy\n", + "test_dir = Path(mkdtemp())\n", + "test_file = test_dir / \"in.txt\"\n", + "with open(test_file, \"w\") as f:\n", + " f.write(\"Contents to be copied\")\n", + "\n", + "# Parameterise the task spec\n", + "cp = Cp(in_file=test_file, destination=test_dir / \"out.txt\")\n", + "\n", + "# Print the cmdline to be run to double check\n", + "print(f\"Command-line to be run: {cp.cmdline}\")\n", + "\n", + "# Run the shell-comand task\n", + "result = cp()\n", + "\n", + "print(\n", + " f\"Contents of copied file ('{result.output.destination}'): \"\n", + " f\"'{Path(result.output.destination).read_text()}'\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If paths to output files are not provided in the parameterisation, it will default to the name of the field" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cp /var/folders/mz/yn83q2fd3s758w1j75d2nnw80000gn/T/tmpoyx19gql/in.txt /Users/tclose/git/workflows/pydra/docs/source/tutorial/destination\n" + ] + } + ], + "source": [ + "cp = Cp(in_file=test_file)\n", + "print(cp.cmdline)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Defifying types\n", + "\n", + "By default, shell-command fields are considered to be of `fileformats.generic.FsObject` type. However, more specific file formats or built-in Python types can be specified by appending the type to the field name after a `:`.\n", + "\n", + "File formats are specified by their MIME type or \"MIME-like\" strings (see the [FileFormats docs](https://arcanaframework.github.io/fileformats/mime.html) for details)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "trim-png /mock/png.png /path/to/output.png\n" + ] + } + ], + "source": [ + "from fileformats.image import Png\n", + "\n", + "TrimPng = shell.define(\"trim-png \")\n", + "\n", + "trim_png = TrimPng(in_image=Png.mock(), out_image=\"/path/to/output.png\")\n", + "\n", + "print(trim_png.cmdline)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Flags and options\n", + "\n", + "Command line flags can also be added to the shell template, either the single or double hyphen form. The field template name immediately following the flag will be associate with that flag.\n", + "\n", + "If there is no space between the flag and the field template, then the field is assumed to be a boolean, otherwise it is assumed to be of type string unless otherwise specified.\n", + "\n", + "If a field is optional, the field template should end with a `?`. Tuple fields are specified by comma separated types.\n", + "\n", + "Varargs are specified by the type followed by an ellipsis, e.g. ``" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'executable': arg(name='executable', type=typing.Union[str, typing.Sequence[str]], default='cp', help_string=\"the first part of the command, can be a string, e.g. 'ls', or a list, e.g. ['ls', '-l', 'dirname']\", requires=[], converter=None, validator=, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=0, sep=None, allowed_values=None, container_path=False, formatter=None),\n", + " 'in_fs_objects': arg(name='in_fs_objects', type=pydra.utils.typing.MultiInputObj[fileformats.generic.fsobject.FsObject], default=EMPTY, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=1, sep=' ', allowed_values=None, container_path=False, formatter=None),\n", + " 'int_arg': arg(name='int_arg', type=int | None, default=None, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='--int-arg', position=5, sep=None, allowed_values=None, container_path=False, formatter=None),\n", + " 'out_dir': outarg(name='out_dir', type=, default=EMPTY, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=2, sep=None, allowed_values=None, container_path=False, formatter=None, path_template='out_dir', keep_extension=False),\n", + " 'recursive': arg(name='recursive', type=, default=False, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='-R', position=3, sep=None, allowed_values=None, container_path=False, formatter=None),\n", + " 'text_arg': arg(name='text_arg', type=str | None, default=None, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='--text-arg', position=4, sep=None, allowed_values=None, container_path=False, formatter=None),\n", + " 'tuple_arg': arg(name='tuple_arg', type=tuple[int, str] | None, default=None, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='--tuple-arg', position=6, sep=None, allowed_values=None, container_path=False, formatter=None)}\n", + "{'out_dir': outarg(name='out_dir', type=, default=EMPTY, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=2, sep=None, allowed_values=None, container_path=False, formatter=None, path_template='out_dir', keep_extension=False),\n", + " 'return_code': out(name='return_code', type=, default=EMPTY, help_string=\"The process' exit code.\", requires=[], converter=None, validator=None, callable=None),\n", + " 'stderr': out(name='stderr', type=, default=EMPTY, help_string='The standard error stream produced by the command.', requires=[], converter=None, validator=None, callable=None),\n", + " 'stdout': out(name='stdout', type=, default=EMPTY, help_string='The standard output stream produced by the command.', requires=[], converter=None, validator=None, callable=None)}\n" + ] + } + ], + "source": [ + "from pprint import pprint\n", + "from pydra.engine.helpers import fields_dict\n", + "\n", + "Cp = shell.define(\n", + " (\n", + " \"cp \"\n", + " \"-R \"\n", + " \"--text-arg \"\n", + " \"--int-arg \"\n", + " \"--tuple-arg \"\n", + " ),\n", + " )\n", + "\n", + "pprint(fields_dict(Cp))\n", + "pprint(fields_dict(Cp.Outputs))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Defaults\n", + "\n", + "Defaults can be specified by appending them to the field template after `=`" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "'--int-arg' default: 99\n" + ] + } + ], + "source": [ + "Cp = shell.define(\n", + " (\n", + " \"cp \"\n", + " \"-R \"\n", + " \"--text-arg \"\n", + " \"--int-arg \"\n", + " \"--tuple-arg \"\n", + " ),\n", + " )\n", + "\n", + "print(f\"'--int-arg' default: {fields_dict(Cp)['int_arg'].default}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Additional field attributes\n", + "\n", + "Additional attributes of the fields in the template can be specified by providing `shell.arg` or `shell.outarg` fields to the `inputs` and `outputs` keyword arguments to the define" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'executable': arg(name='executable', type=typing.Union[str, typing.Sequence[str]], default='cp', help_string=\"the first part of the command, can be a string, e.g. 'ls', or a list, e.g. ['ls', '-l', 'dirname']\", requires=[], converter=None, validator=, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=0, sep=None, allowed_values=None, container_path=False, formatter=None),\n", + " 'in_fs_objects': arg(name='in_fs_objects', type=pydra.utils.typing.MultiInputObj[fileformats.generic.fsobject.FsObject], default=EMPTY, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=1, sep=' ', allowed_values=None, container_path=False, formatter=None),\n", + " 'int_arg': arg(name='int_arg', type=int | None, default=None, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='--int-arg', position=4, sep=None, allowed_values=None, container_path=False, formatter=None),\n", + " 'out_dir': outarg(name='out_dir', type=, default=EMPTY, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=-2, sep=None, allowed_values=None, container_path=False, formatter=None, path_template='out_dir', keep_extension=False),\n", + " 'out_file': outarg(name='out_file', type=fileformats.generic.file.File | None, default=None, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=-1, sep=None, allowed_values=None, container_path=False, formatter=None, path_template='out_file', keep_extension=False),\n", + " 'recursive': arg(name='recursive', type=, default=False, help_string='If source_file designates a directory, cp copies the directory and the entire subtree connected at that point.', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='-R', position=2, sep=None, allowed_values=None, container_path=False, formatter=None),\n", + " 'text_arg': arg(name='text_arg', type=, default=EMPTY, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='--text-arg', position=3, sep=None, allowed_values=None, container_path=False, formatter=None),\n", + " 'tuple_arg': arg(name='tuple_arg', type=tuple[int, str], default=EMPTY, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='--tuple-arg', position=5, sep=None, allowed_values=None, container_path=False, formatter=None)}\n", + "{'out_dir': outarg(name='out_dir', type=, default=EMPTY, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=-2, sep=None, allowed_values=None, container_path=False, formatter=None, path_template='out_dir', keep_extension=False),\n", + " 'out_file': outarg(name='out_file', type=fileformats.generic.file.File | None, default=None, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=-1, sep=None, allowed_values=None, container_path=False, formatter=None, path_template='out_file', keep_extension=False),\n", + " 'return_code': out(name='return_code', type=, default=EMPTY, help_string=\"The process' exit code.\", requires=[], converter=None, validator=None, callable=None),\n", + " 'stderr': out(name='stderr', type=, default=EMPTY, help_string='The standard error stream produced by the command.', requires=[], converter=None, validator=None, callable=None),\n", + " 'stdout': out(name='stdout', type=, default=EMPTY, help_string='The standard output stream produced by the command.', requires=[], converter=None, validator=None, callable=None)}\n" + ] + } + ], + "source": [ + "Cp = shell.define(\n", + " (\n", + " \"cp \"\n", + " \"-R \"\n", + " \"--text-arg \"\n", + " \"--int-arg \"\n", + " \"--tuple-arg \"\n", + " ),\n", + " inputs={\"recursive\": shell.arg(\n", + " help_string=(\n", + " \"If source_file designates a directory, cp copies the directory and \"\n", + " \"the entire subtree connected at that point.\"\n", + " )\n", + " )},\n", + " outputs={\n", + " \"out_dir\": shell.outarg(position=-2),\n", + " \"out_file\": shell.outarg(position=-1),\n", + " },\n", + " )\n", + "\n", + "\n", + "pprint(fields_dict(Cp))\n", + "pprint(fields_dict(Cp.Outputs))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Callable outptus\n", + "\n", + "In addition to outputs that are specified to the tool on the command line, outputs can be derived from the outputs of the tool by providing a Python function that can take the output directory and inputs as arguments and return the output value. Callables can be either specified in the `callable` attribute of the `shell.out` field, or in a dictionary mapping the output name to the callable" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Size of the output file is: 256\n" + ] + } + ], + "source": [ + "import os\n", + "from pydra.design import shell\n", + "from pathlib import Path\n", + "from fileformats.generic import File\n", + "\n", + "# Arguments to the callable function can be one of \n", + "def get_file_size(out_file: Path) -> int:\n", + " \"\"\"Calculate the file size\"\"\"\n", + " result = os.stat(out_file)\n", + " return result.st_size\n", + "\n", + "\n", + "CpWithSize = shell.define(\n", + " \"cp \",\n", + " outputs={\"out_file_size\": get_file_size},\n", + ")\n", + "\n", + "# Parameterise the task spec\n", + "cp_with_size = CpWithSize(in_file=File.sample())\n", + "\n", + "# Run the command\n", + "result = cp_with_size()\n", + "\n", + "\n", + "print(f\"Size of the output file is: {result.output.out_file_size}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The callable can take any combination of the following arguments, which will be passed\n", + "to it when it is called\n", + "\n", + "* field: the `Field` object to be provided a value, useful when writing generic callables\n", + "* output_dir: a `Path` object referencing the working directory the command was run within\n", + "* inputs: a dictionary containing all the resolved inputs to the task\n", + "* stdout: the standard output stream produced by the command\n", + "* stderr: the standard error stream produced by the command\n", + "* *name of an input*: the name of any of the input arguments to the task, including output args that are part of the command line (i.e. output files)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Dataclass form\n", + "\n", + "Like with Python tasks, shell-tasks can also be specified in dataclass-form by using `shell.define` as a decorator." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'executable': arg(name='executable', type=typing.Union[str, typing.Sequence[str]], default='cp', help_string=\"the first part of the command, can be a string, e.g. 'ls', or a list, e.g. ['ls', '-l', 'dirname']\", requires=[], converter=None, validator=, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=0, sep=None, allowed_values=None, container_path=False, formatter=None),\n", + " 'in_fs_objects': arg(name='in_fs_objects', type=pydra.utils.typing.MultiInputObj[fileformats.generic.fsobject.FsObject], default=EMPTY, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=5, sep=None, allowed_values=None, container_path=False, formatter=None),\n", + " 'int_arg': arg(name='int_arg', type=int | None, default=None, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='--int-arg', position=1, sep=None, allowed_values=None, container_path=False, formatter=None),\n", + " 'recursive': arg(name='recursive', type=, default=False, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='-R', position=2, sep=None, allowed_values=None, container_path=False, formatter=None),\n", + " 'text_arg': arg(name='text_arg', type=, default=EMPTY, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='--text-arg', position=3, sep=None, allowed_values=None, container_path=False, formatter=None),\n", + " 'tuple_arg': arg(name='tuple_arg', type=tuple[int, str] | None, default=None, help_string='', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='--tuple-arg', position=4, sep=None, allowed_values=None, container_path=False, formatter=None)}\n", + "{'out_file': out(name='out_file', type=, default=EMPTY, help_string='', requires=[], converter=None, validator=None, callable=None),\n", + " 'out_file_size': out(name='out_file_size', type=, default=EMPTY, help_string='', requires=[], converter=None, validator=None, callable=),\n", + " 'return_code': out(name='return_code', type=, default=EMPTY, help_string=\"The process' exit code.\", requires=[], converter=None, validator=None, callable=None),\n", + " 'stderr': out(name='stderr', type=, default=EMPTY, help_string='The standard error stream produced by the command.', requires=[], converter=None, validator=None, callable=None),\n", + " 'stdout': out(name='stdout', type=, default=EMPTY, help_string='The standard output stream produced by the command.', requires=[], converter=None, validator=None, callable=None)}\n" + ] + } + ], + "source": [ + "from fileformats.generic import FsObject, Directory\n", + "from pydra.utils.typing import MultiInputObj\n", + "\n", + "@shell.define\n", + "class CpWithSize:\n", + "\n", + " executable = \"cp\"\n", + "\n", + " in_fs_objects: MultiInputObj[FsObject]\n", + " recursive: bool = shell.arg(argstr=\"-R\")\n", + " text_arg: str = shell.arg(argstr=\"--text-arg\")\n", + " int_arg: int | None = shell.arg(argstr=\"--int-arg\")\n", + " tuple_arg: tuple[int, str] | None = shell.arg(argstr=\"--tuple-arg\")\n", + "\n", + " class Outputs:\n", + " out_file: File\n", + " out_file_size: int = shell.out(callable=get_file_size)\n", + "\n", + "\n", + "pprint(fields_dict(CpWithSize))\n", + "pprint(fields_dict(CpWithSize.Outputs))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To make workflows that use the interface type-checkable, the canonical form of a shell\n", + "task dataclass should inherit from `shell.Def` parameterized by its nested Outputs class,\n", + "and the `Outputs` nested class should inherit from `shell.Outputs`." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "from pydra.engine.specs import ShellDef, ShellOutputs\n", + "\n", + "@shell.define\n", + "class Cp(ShellDef[\"Cp.Outputs\"]):\n", + "\n", + " executable = \"cp\"\n", + "\n", + " in_fs_objects: MultiInputObj[FsObject]\n", + " recursive: bool = shell.arg(argstr=\"-R\", default=False)\n", + " text_arg: str = shell.arg(argstr=\"--text-arg\")\n", + " int_arg: int | None = shell.arg(argstr=\"--int-arg\")\n", + " tuple_arg: tuple[int, str] | None = shell.arg(argstr=\"--tuple-arg\")\n", + "\n", + " @shell.outputs\n", + " class Outputs(ShellOutputs):\n", + " out_dir: Directory = shell.outarg(path_template=\"{out_dir}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Dynamic definitions\n", + "\n", + "In some cases, it is required to generate the definition for a task dynamically, which can be done by just providing the executable to `shell.define` and specifying all inputs and outputs explicitly" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ACommand input fields: [arg(name='in_file', type=, default=EMPTY, help_string='output file', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=-2, sep=None, allowed_values=None, container_path=False, formatter=None), outarg(name='out_file', type=, default=EMPTY, help_string='output file', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=-1, sep=None, allowed_values=None, container_path=False, formatter=None, path_template=None, keep_extension=False), arg(name='executable', type=typing.Union[str, typing.Sequence[str]], default='a-command', help_string=\"the first part of the command, can be a string, e.g. 'ls', or a list, e.g. ['ls', '-l', 'dirname']\", requires=[], converter=None, validator=, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=0, sep=None, allowed_values=None, container_path=False, formatter=None)]\n", + "ACommand input fields: [outarg(name='out_file', type=, default=EMPTY, help_string='output file', requires=[], converter=None, validator=None, xor=(), copy_mode=, copy_collation=, copy_ext_decomp=, readonly=False, argstr='', position=-1, sep=None, allowed_values=None, container_path=False, formatter=None, path_template=None, keep_extension=False), out(name='out_file_size', type=, default=EMPTY, help_string='size of the output directory', requires=[], converter=None, validator=None, callable=), out(name='return_code', type=, default=EMPTY, help_string=\"The process' exit code.\", requires=[], converter=None, validator=None, callable=None), out(name='stdout', type=, default=EMPTY, help_string='The standard output stream produced by the command.', requires=[], converter=None, validator=None, callable=None), out(name='stderr', type=, default=EMPTY, help_string='The standard error stream produced by the command.', requires=[], converter=None, validator=None, callable=None)]\n" + ] + } + ], + "source": [ + "from fileformats.generic import File\n", + "from pydra.engine.helpers import list_fields\n", + "\n", + "ACommand = shell.define(\n", + " \"a-command\",\n", + " inputs={\n", + " \"in_file\": shell.arg(type=File, help_string=\"output file\", argstr=\"\", position=-2)\n", + " },\n", + " outputs={\n", + " \"out_file\": shell.outarg(\n", + " type=File, help_string=\"output file\", argstr=\"\", position=-1\n", + " ),\n", + " \"out_file_size\": {\n", + " \"type\": int,\n", + " \"help_string\": \"size of the output directory\",\n", + " \"callable\": get_file_size,\n", + " }\n", + " },\n", + ")\n", + "\n", + "\n", + "print(f\"ACommand input fields: {list_fields(ACommand)}\")\n", + "print(f\"ACommand input fields: {list_fields(ACommand.Outputs)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "wf12", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/new-docs/source/tutorial/workflow.ipynb b/new-docs/source/tutorial/workflow.ipynb index e6c812e13..46e14099e 100644 --- a/new-docs/source/tutorial/workflow.ipynb +++ b/new-docs/source/tutorial/workflow.ipynb @@ -1,485 +1,485 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Workflows\n", - "\n", - "In Pydra, workflows are DAG of component tasks to be executed on specified inputs.\n", - "Workflow specifications are dataclasses, which interchangeable with Python and shell tasks\n", - "specifications and executed in the same way." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Constructor functions\n", - "\n", - "Workflows are typically defined using the `pydra.design.workflow.define` decorator on \n", - "a \"constructor\" function that generates the workflow. For example, given two task\n", - "specifications, `Add` and `Mul`." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from pydra.design import workflow, python\n", - "\n", - "# Example python task specifications\n", - "@python.define\n", - "def Add(a, b):\n", - " return a + b\n", - "\n", - "\n", - "@python.define\n", - "def Mul(a, b):\n", - " return a * b" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - " we can create a simple workflow specification using `workflow.define` to decorate a function that constructs the workflow. Nodes are added to the workflow being constructed by calling `workflow.add` function." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "@workflow.define\n", - "def BasicWorkflow(a, b):\n", - " add = workflow.add(Add(a=a, b=b))\n", - " mul = workflow.add(Mul(a=add.out, b=b))\n", - " return mul.out" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "`workflow.add` returns an \"outputs\" object corresponding to the specification added to the workflow. The fields of the outptus object can be referenced as inputs to downstream workflow nodes. Note that these fields are just placeholders for the values that will be returned and can't be used in conditional statements during workflow construction. The return value(s) of workflow constructor function are the placeholders of the fields that are to be the outputs of the workflow.\n", - "\n", - "It is also possible to define new tasks to add to the workflow inline the constructor and type the inputs and outputs of the workflow." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "from pydra.design import shell\n", - "from fileformats import image, video\n", - "\n", - "@workflow.define\n", - "def ShellWorkflow(\n", - " input_video: video.Mp4,\n", - " watermark: image.Png,\n", - " watermark_dims: tuple[int, int] = (10, 10),\n", - ") -> video.Mp4:\n", - "\n", - " add_watermark = workflow.add(\n", - " shell.define(\n", - " \"ffmpeg -i -i \"\n", - " \"-filter_complex \"\n", - " )(\n", - " in_video=input_video,\n", - " watermark=watermark,\n", - " filter=\"overlay={}:{}\".format(*watermark_dims),\n", - " )\n", - " )\n", - " output_video = workflow.add(\n", - " shell.define(\n", - " \"HandBrakeCLI -i -o \"\n", - " \"--width --height \",\n", - " )(in_video=add_watermark.out_video, width=1280, height=720)\n", - " ).out_video\n", - "\n", - " return output_video # test implicit detection of output name" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Accessing the workflow object\n", - "\n", - "If you need to access the workflow object being constructed from inside the constructor function you can use `workflow.this()`." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "@python.define(outputs=[\"divided\"])\n", - "def Divide(x, y):\n", - " return x / y\n", - "\n", - "\n", - "@workflow.define(outputs=[\"out1\", \"out2\"])\n", - "def DirectAccesWorkflow(a: int, b: float) -> tuple[float, float]:\n", - " \"\"\"A test workflow demonstration a few alternative ways to set and connect nodes\n", - "\n", - " Args:\n", - " a: An integer input\n", - " b: A float input\n", - "\n", - " Returns:\n", - " out1: The first output\n", - " out2: The second output\n", - " \"\"\"\n", - "\n", - " wf = workflow.this()\n", - "\n", - " add = wf.add(Add(x=a, y=b), name=\"addition\")\n", - " mul = wf.add(python.define(Mul, outputs={\"out\": float})(x=add.z, y=b))\n", - " divide = wf.add(Divide(x=wf[\"addition\"].lzout.z, y=mul.out), name=\"division\")\n", - "\n", - " # Alter one of the inputs to a node after it has been initialised\n", - " wf[\"Mul\"].inputs.y *= 2\n", - "\n", - " return mul.out, divide.divided" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Directly access the workflow being constructed also enables you to set the outputs of the workflow directly" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "@workflow.define(outputs={\"out1\": float, \"out2\": float})\n", - "def SetOutputsOfWorkflow(a: int, b: float):\n", - " \"\"\"A test workflow demonstration a few alternative ways to set and connect nodes\n", - "\n", - " Args:\n", - " a: An integer input\n", - " b: A float input\n", - "\n", - " Returns:\n", - " out1: The first output\n", - " out2: The second output\n", - " \"\"\"\n", - "\n", - " wf = workflow.this()\n", - "\n", - " add = wf.add(Add(x=a, y=b), name=\"addition\")\n", - " mul = wf.add(python.define(Mul, outputs={\"out\": float})(x=add.z, y=b))\n", - " divide = wf.add(Divide(x=wf[\"addition\"].lzout.z, y=mul.out), name=\"division\")\n", - "\n", - " # Alter one of the inputs to a node after it has been initialised\n", - " wf[\"Mul\"].inputs.y *= 2\n", - "\n", - " # Set the outputs of the workflow directly\n", - " wf.outputs.out1 = mul.out\n", - " wf.outputs.out2 = divide.divided" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Dataclass form\n", - "\n", - "Like with Python and shell tasks, it is also possible to specify workflows in \"dataclass form\" in order to be more explicit to linters, which can be worth the extra effort when creating a suite of workflows to be shared publicly. In this case the workflow constructor should be a static method of the dataclasss named `constructor`.\n", - "\n", - "This form also lends itself to defining custom converters and validators on the fields" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "from pydra.engine.specs import WorkflowSpec, WorkflowOutputs\n", - "\n", - "def a_converter(value):\n", - " if value is None:\n", - " return value\n", - " return float(value)\n", - "\n", - "@workflow.define\n", - "class LibraryWorkflow(WorkflowSpec[\"MyLibraryWorkflow.Outputs\"]):\n", - "\n", - " a: int\n", - " b: float = workflow.arg(\n", - " help_string=\"A float input\",\n", - " converter=a_converter,\n", - " )\n", - "\n", - " @staticmethod\n", - " def constructor(a, b):\n", - " add = workflow.add(Add(a=a, b=b))\n", - " mul = workflow.add(Mul(a=add.out, b=b))\n", - " return mul.out\n", - "\n", - " @workflow.outputs\n", - " class Outputs(WorkflowOutputs):\n", - " out: float" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Splitting/combining task inputs\n", - "\n", - "Sometimes, you might want to perform the same task over a set of input values/files, and then collect the results into a list to perform further processing. This can be achieved by using the `split` and `combine` methods" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "@python.define\n", - "def Sum(x: list[float]) -> float:\n", - " return sum(x)\n", - "\n", - "@workflow.define\n", - "def SplitWorkflow(a: list[int], b: list[float]) -> list[float]:\n", - " # Multiply over all combinations of the elements of a and b, then combine the results\n", - " # for each a element into a list over each b element\n", - " mul = workflow.add(Mul()).split(x=a, y=b).combine(\"x\")\n", - " # Sume the multiplications across all all b elements for each a element\n", - " sum = workflow.add(Sum(x=mul.out))\n", - " return sum.out" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The combination step doesn't have to be done on the same step as the split, in which case the splits propagate to downstream nodes" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "@workflow.define\n", - "def SplitThenCombineWorkflow(a: list[int], b: list[float], c: float) -> list[float]:\n", - " mul = workflow.add(Mul()).split(x=a, y=b)\n", - " add = workflow.add(Add(x=mul.out, y=c)).combine(\"Mul.x\")\n", - " sum = workflow.add(Sum(x=add.out))\n", - " return sum.out" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For more advanced discussion on the intricacies of splitting and combining see [Splitting and combining](../explanation/splitting-combining.html)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Nested and conditional workflows\n", - "\n", - "One of the most powerful features of Pydra is the ability to use inline Python code to conditionally add/omit nodes to workflow, and alter the parameterisation of the nodes, depending on inputs to the workflow " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "@workflow.define\n", - "def ConditionalWorkflow(\n", - " input_video: video.Mp4,\n", - " watermark: image.Png,\n", - " watermark_dims: tuple[int, int] | None = None,\n", - ") -> video.Mp4:\n", - "\n", - " if watermark_dims is not None:\n", - " add_watermark = workflow.add(\n", - " shell.define(\n", - " \"ffmpeg -i -i \"\n", - " \"-filter_complex \"\n", - " )(\n", - " in_video=input_video,\n", - " watermark=watermark,\n", - " filter=\"overlay={}:{}\".format(*watermark_dims),\n", - " )\n", - " )\n", - " handbrake_input = add_watermark.out_video\n", - " else:\n", - " handbrake_input = input_video\n", - "\n", - " output_video = workflow.add(\n", - " shell.define(\n", - " \"HandBrakeCLI -i -o \"\n", - " \"--width --height \",\n", - " )(in_video=handbrake_input, width=1280, height=720)\n", - " ).out_video\n", - "\n", - " return output_video # test implicit detection of output name" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note that outputs of upstream nodes cannot be used in conditional statements, since these are just placeholders at the time the workflow is being constructed. However, you can get around\n", - "this limitation by placing the conditional logic within a nested workflow" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "@python.define\n", - "def Subtract(x: float, y: float) -> float:\n", - " return x - y\n", - "\n", - "@workflow.define\n", - "def RecursiveNestedWorkflow(a: float, depth: int) -> float:\n", - " add = workflow.add(Add(x=a, y=1))\n", - " decrement_depth = workflow.add(Subtract(x=depth, y=1))\n", - " if depth > 0:\n", - " out_node = workflow.add(\n", - " RecursiveNestedWorkflow(a=add.out, depth=decrement_depth.out)\n", - " )\n", - " else:\n", - " out_node = add\n", - " return out_node.out" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For more detailed discussion of the construction of conditional workflows and \"lazy field\"\n", - "placeholders see [Conditionals and lazy fields](../explanation/conditional-lazy.html)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Type-checking between nodes\n", - "\n", - "Pydra utilizes Python type annotations to implement strong type-checking, which is performed\n", - "when values or upstream outputs are assigned to task specification inputs.\n", - "\n", - "Task input and output fields do not need to be assigned types, since they will default to `typing.Any`.\n", - "However, if they are assigned a type and a value or output from an upstream node conflicts\n", - "with the type, a `TypeError` will be raised at construction time.\n", - "\n", - "Note that the type-checking \"assumes the best\", and will pass if the upstream field is typed\n", - "by `Any` or a super-class of the field being assigned to. For example, an input of\n", - "`fileformats.generic.File` passed to a field expecting a `fileformats.image.Png` file type,\n", - "because `Png` is a subtype of `File`, where as `fileformats.image.Jpeg` input would fail\n", - "since it is clearly not the intended type.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "from fileformats import generic\n", - "\n", - "Mp4Handbrake = shell.define(\n", - " \"HandBrakeCLI -i -o \"\n", - " \"--width --height \",\n", - ")\n", - "\n", - "\n", - "QuicktimeHandbrake = shell.define(\n", - " \"HandBrakeCLI -i -o \"\n", - " \"--width --height \",\n", - ")\n", - "\n", - "@workflow.define\n", - "def TypeErrorWorkflow(\n", - " input_video: video.Mp4,\n", - " watermark: generic.File,\n", - " watermark_dims: tuple[int, int] = (10, 10),\n", - ") -> video.Mp4:\n", - "\n", - " add_watermark = workflow.add(\n", - " shell.define(\n", - " \"ffmpeg -i -i \"\n", - " \"-filter_complex \"\n", - " )(\n", - " in_video=input_video, # This is OK because in_video is typed Any\n", - " watermark=watermark, # Type is OK because generic.File is superclass of image.Png\n", - " filter=\"overlay={}:{}\".format(*watermark_dims),\n", - " ),\n", - " name=\"add_watermark\",\n", - " )\n", - "\n", - " try:\n", - " handbrake = workflow.add(\n", - " QuicktimeHandbrake(in_video=add_watermark.out_video, width=1280, height=720),\n", - " ) # This will raise a TypeError because the input video is an Mp4\n", - " except TypeError:\n", - " handbrake = workflow.add(\n", - " Mp4Handbrake(in_video=add_watermark.out_video, width=1280, height=720),\n", - " ) # The type of the input video is now correct\n", - "\n", - " return handbrake.output_video" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For more detailed discussion on Pydra's type-checking see [Type Checking](../explanation/typing.html)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "wf12", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.5" - } - }, - "nbformat": 4, - "nbformat_minor": 2 + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Workflows\n", + "\n", + "In Pydra, workflows are DAG of component tasks to be executed on specified inputs.\n", + "Workflow definitions are dataclasses, which interchangeable with Python and shell tasks\n", + "definitions and executed in the same way." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Constructor functions\n", + "\n", + "Workflows are typically defined using the `pydra.design.workflow.define` decorator on \n", + "a \"constructor\" function that generates the workflow. For example, given two task\n", + "definitions, `Add` and `Mul`." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from pydra.design import workflow, python\n", + "\n", + "# Example python task definitions\n", + "@python.define\n", + "def Add(a, b):\n", + " return a + b\n", + "\n", + "\n", + "@python.define\n", + "def Mul(a, b):\n", + " return a * b" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + " we can create a simple workflow definition using `workflow.define` to decorate a function that constructs the workflow. Nodes are added to the workflow being constructed by calling `workflow.add` function." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "@workflow.define\n", + "def BasicWorkflow(a, b):\n", + " add = workflow.add(Add(a=a, b=b))\n", + " mul = workflow.add(Mul(a=add.out, b=b))\n", + " return mul.out" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`workflow.add` returns an \"outputs\" object corresponding to the definition added to the workflow. The fields of the outptus object can be referenced as inputs to downstream workflow nodes. Note that these fields are just placeholders for the values that will be returned and can't be used in conditional statements during workflow construction. The return value(s) of workflow constructor function are the placeholders of the fields that are to be the outputs of the workflow.\n", + "\n", + "It is also possible to define new tasks to add to the workflow inline the constructor and type the inputs and outputs of the workflow." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from pydra.design import shell\n", + "from fileformats import image, video\n", + "\n", + "@workflow.define\n", + "def ShellWorkflow(\n", + " input_video: video.Mp4,\n", + " watermark: image.Png,\n", + " watermark_dims: tuple[int, int] = (10, 10),\n", + ") -> video.Mp4:\n", + "\n", + " add_watermark = workflow.add(\n", + " shell.define(\n", + " \"ffmpeg -i -i \"\n", + " \"-filter_complex \"\n", + " )(\n", + " in_video=input_video,\n", + " watermark=watermark,\n", + " filter=\"overlay={}:{}\".format(*watermark_dims),\n", + " )\n", + " )\n", + " output_video = workflow.add(\n", + " shell.define(\n", + " \"HandBrakeCLI -i -o \"\n", + " \"--width --height \",\n", + " )(in_video=add_watermark.out_video, width=1280, height=720)\n", + " ).out_video\n", + "\n", + " return output_video # test implicit detection of output name" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Accessing the workflow object\n", + "\n", + "If you need to access the workflow object being constructed from inside the constructor function you can use `workflow.this()`." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "@python.define(outputs=[\"divided\"])\n", + "def Divide(x, y):\n", + " return x / y\n", + "\n", + "\n", + "@workflow.define(outputs=[\"out1\", \"out2\"])\n", + "def DirectAccesWorkflow(a: int, b: float) -> tuple[float, float]:\n", + " \"\"\"A test workflow demonstration a few alternative ways to set and connect nodes\n", + "\n", + " Args:\n", + " a: An integer input\n", + " b: A float input\n", + "\n", + " Returns:\n", + " out1: The first output\n", + " out2: The second output\n", + " \"\"\"\n", + "\n", + " wf = workflow.this()\n", + "\n", + " add = wf.add(Add(x=a, y=b), name=\"addition\")\n", + " mul = wf.add(python.define(Mul, outputs={\"out\": float})(x=add.z, y=b))\n", + " divide = wf.add(Divide(x=wf[\"addition\"].lzout.z, y=mul.out), name=\"division\")\n", + "\n", + " # Alter one of the inputs to a node after it has been initialised\n", + " wf[\"Mul\"].inputs.y *= 2\n", + "\n", + " return mul.out, divide.divided" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Directly access the workflow being constructed also enables you to set the outputs of the workflow directly" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "@workflow.define(outputs={\"out1\": float, \"out2\": float})\n", + "def SetOutputsOfWorkflow(a: int, b: float):\n", + " \"\"\"A test workflow demonstration a few alternative ways to set and connect nodes\n", + "\n", + " Args:\n", + " a: An integer input\n", + " b: A float input\n", + "\n", + " Returns:\n", + " out1: The first output\n", + " out2: The second output\n", + " \"\"\"\n", + "\n", + " wf = workflow.this()\n", + "\n", + " add = wf.add(Add(x=a, y=b), name=\"addition\")\n", + " mul = wf.add(python.define(Mul, outputs={\"out\": float})(x=add.z, y=b))\n", + " divide = wf.add(Divide(x=wf[\"addition\"].lzout.z, y=mul.out), name=\"division\")\n", + "\n", + " # Alter one of the inputs to a node after it has been initialised\n", + " wf[\"Mul\"].inputs.y *= 2\n", + "\n", + " # Set the outputs of the workflow directly\n", + " wf.outputs.out1 = mul.out\n", + " wf.outputs.out2 = divide.divided" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Dataclass form\n", + "\n", + "Like with Python and shell tasks, it is also possible to specify workflows in \"dataclass form\" in order to be more explicit to linters, which can be worth the extra effort when creating a suite of workflows to be shared publicly. In this case the workflow constructor should be a static method of the dataclasss named `constructor`.\n", + "\n", + "This form also lends itself to defining custom converters and validators on the fields" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "from pydra.engine.specs import WorkflowDef, WorkflowOutputs\n", + "\n", + "def a_converter(value):\n", + " if value is None:\n", + " return value\n", + " return float(value)\n", + "\n", + "@workflow.define\n", + "class LibraryWorkflow(WorkflowDef[\"MyLibraryWorkflow.Outputs\"]):\n", + "\n", + " a: int\n", + " b: float = workflow.arg(\n", + " help_string=\"A float input\",\n", + " converter=a_converter,\n", + " )\n", + "\n", + " @staticmethod\n", + " def constructor(a, b):\n", + " add = workflow.add(Add(a=a, b=b))\n", + " mul = workflow.add(Mul(a=add.out, b=b))\n", + " return mul.out\n", + "\n", + " @workflow.outputs\n", + " class Outputs(WorkflowOutputs):\n", + " out: float" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Splitting/combining task inputs\n", + "\n", + "Sometimes, you might want to perform the same task over a set of input values/files, and then collect the results into a list to perform further processing. This can be achieved by using the `split` and `combine` methods" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "@python.define\n", + "def Sum(x: list[float]) -> float:\n", + " return sum(x)\n", + "\n", + "@workflow.define\n", + "def SplitWorkflow(a: list[int], b: list[float]) -> list[float]:\n", + " # Multiply over all combinations of the elements of a and b, then combine the results\n", + " # for each a element into a list over each b element\n", + " mul = workflow.add(Mul()).split(x=a, y=b).combine(\"x\")\n", + " # Sume the multiplications across all all b elements for each a element\n", + " sum = workflow.add(Sum(x=mul.out))\n", + " return sum.out" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The combination step doesn't have to be done on the same step as the split, in which case the splits propagate to downstream nodes" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "@workflow.define\n", + "def SplitThenCombineWorkflow(a: list[int], b: list[float], c: float) -> list[float]:\n", + " mul = workflow.add(Mul()).split(x=a, y=b)\n", + " add = workflow.add(Add(x=mul.out, y=c)).combine(\"Mul.x\")\n", + " sum = workflow.add(Sum(x=add.out))\n", + " return sum.out" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For more advanced discussion on the intricacies of splitting and combining see [Splitting and combining](../explanation/splitting-combining.html)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Nested and conditional workflows\n", + "\n", + "One of the most powerful features of Pydra is the ability to use inline Python code to conditionally add/omit nodes to workflow, and alter the parameterisation of the nodes, depending on inputs to the workflow " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@workflow.define\n", + "def ConditionalWorkflow(\n", + " input_video: video.Mp4,\n", + " watermark: image.Png,\n", + " watermark_dims: tuple[int, int] | None = None,\n", + ") -> video.Mp4:\n", + "\n", + " if watermark_dims is not None:\n", + " add_watermark = workflow.add(\n", + " shell.define(\n", + " \"ffmpeg -i -i \"\n", + " \"-filter_complex \"\n", + " )(\n", + " in_video=input_video,\n", + " watermark=watermark,\n", + " filter=\"overlay={}:{}\".format(*watermark_dims),\n", + " )\n", + " )\n", + " handbrake_input = add_watermark.out_video\n", + " else:\n", + " handbrake_input = input_video\n", + "\n", + " output_video = workflow.add(\n", + " shell.define(\n", + " \"HandBrakeCLI -i -o \"\n", + " \"--width --height \",\n", + " )(in_video=handbrake_input, width=1280, height=720)\n", + " ).out_video\n", + "\n", + " return output_video # test implicit detection of output name" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that outputs of upstream nodes cannot be used in conditional statements, since these are just placeholders at the time the workflow is being constructed. However, you can get around\n", + "this limitation by placing the conditional logic within a nested workflow" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "@python.define\n", + "def Subtract(x: float, y: float) -> float:\n", + " return x - y\n", + "\n", + "@workflow.define\n", + "def RecursiveNestedWorkflow(a: float, depth: int) -> float:\n", + " add = workflow.add(Add(x=a, y=1))\n", + " decrement_depth = workflow.add(Subtract(x=depth, y=1))\n", + " if depth > 0:\n", + " out_node = workflow.add(\n", + " RecursiveNestedWorkflow(a=add.out, depth=decrement_depth.out)\n", + " )\n", + " else:\n", + " out_node = add\n", + " return out_node.out" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For more detailed discussion of the construction of conditional workflows and \"lazy field\"\n", + "placeholders see [Conditionals and lazy fields](../explanation/conditional-lazy.html)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Type-checking between nodes\n", + "\n", + "Pydra utilizes Python type annotations to implement strong type-checking, which is performed\n", + "when values or upstream outputs are assigned to task definition inputs.\n", + "\n", + "Task input and output fields do not need to be assigned types, since they will default to `typing.Any`.\n", + "However, if they are assigned a type and a value or output from an upstream node conflicts\n", + "with the type, a `TypeError` will be raised at construction time.\n", + "\n", + "Note that the type-checking \"assumes the best\", and will pass if the upstream field is typed\n", + "by `Any` or a super-class of the field being assigned to. For example, an input of\n", + "`fileformats.generic.File` passed to a field expecting a `fileformats.image.Png` file type,\n", + "because `Png` is a subtype of `File`, where as `fileformats.image.Jpeg` input would fail\n", + "since it is clearly not the intended type.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "from fileformats import generic\n", + "\n", + "Mp4Handbrake = shell.define(\n", + " \"HandBrakeCLI -i -o \"\n", + " \"--width --height \",\n", + ")\n", + "\n", + "\n", + "QuicktimeHandbrake = shell.define(\n", + " \"HandBrakeCLI -i -o \"\n", + " \"--width --height \",\n", + ")\n", + "\n", + "@workflow.define\n", + "def TypeErrorWorkflow(\n", + " input_video: video.Mp4,\n", + " watermark: generic.File,\n", + " watermark_dims: tuple[int, int] = (10, 10),\n", + ") -> video.Mp4:\n", + "\n", + " add_watermark = workflow.add(\n", + " shell.define(\n", + " \"ffmpeg -i -i \"\n", + " \"-filter_complex \"\n", + " )(\n", + " in_video=input_video, # This is OK because in_video is typed Any\n", + " watermark=watermark, # Type is OK because generic.File is superclass of image.Png\n", + " filter=\"overlay={}:{}\".format(*watermark_dims),\n", + " ),\n", + " name=\"add_watermark\",\n", + " )\n", + "\n", + " try:\n", + " handbrake = workflow.add(\n", + " QuicktimeHandbrake(in_video=add_watermark.out_video, width=1280, height=720),\n", + " ) # This will raise a TypeError because the input video is an Mp4\n", + " except TypeError:\n", + " handbrake = workflow.add(\n", + " Mp4Handbrake(in_video=add_watermark.out_video, width=1280, height=720),\n", + " ) # The type of the input video is now correct\n", + "\n", + " return handbrake.output_video" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For more detailed discussion on Pydra's type-checking see [Type Checking](../explanation/typing.html)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "wf12", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/pydra/design/base.py b/pydra/design/base.py index dec1be2fc..8e9b42435 100644 --- a/pydra/design/base.py +++ b/pydra/design/base.py @@ -26,7 +26,7 @@ if ty.TYPE_CHECKING: - from pydra.engine.specs import TaskSpec, TaskOutputs + from pydra.engine.specs import TaskDef, TaskOutputs from pydra.engine.core import Task __all__ = [ @@ -75,7 +75,7 @@ class Requirement: name: str allowed_values: list[str] = attrs.field(factory=list, converter=list) - def satisfied(self, inputs: "TaskSpec") -> bool: + def satisfied(self, inputs: "TaskDef") -> bool: """Check if the requirement is satisfied by the inputs""" value = getattr(inputs, self.name) if value is attrs.NOTHING: @@ -122,7 +122,7 @@ class RequirementSet: converter=requirements_converter, ) - def satisfied(self, inputs: "TaskSpec") -> bool: + def satisfied(self, inputs: "TaskDef") -> bool: """Check if all the requirements are satisfied by the inputs""" return all(req.satisfied(inputs) for req in self.requirements) @@ -155,7 +155,7 @@ def requires_converter( @attrs.define(kw_only=True) class Field: - """Base class for input and output fields to task specifications + """Base class for input and output fields to task definitions Parameters ---------- @@ -193,14 +193,14 @@ class Field: converter: ty.Callable | None = None validator: ty.Callable | None = None - def requirements_satisfied(self, inputs: "TaskSpec") -> bool: + def requirements_satisfied(self, inputs: "TaskDef") -> bool: """Check if all the requirements are satisfied by the inputs""" return any(req.satisfied(inputs) for req in self.requires) @attrs.define(kw_only=True) class Arg(Field): - """Base class for input fields of task specifications + """Base class for input fields of task definitions Parameters ---------- @@ -242,7 +242,7 @@ class Arg(Field): @attrs.define(kw_only=True) class Out(Field): - """Base class for output fields of task specifications + """Base class for output fields of task definitions Parameters ---------- @@ -350,7 +350,7 @@ def get_fields(klass, field_type, auto_attribs, helps) -> dict[str, Field]: def make_task_spec( - spec_type: type["TaskSpec"], + spec_type: type["TaskDef"], out_type: type["TaskOutputs"], task_type: type["Task"], inputs: dict[str, Arg], @@ -360,7 +360,7 @@ def make_task_spec( bases: ty.Sequence[type] = (), outputs_bases: ty.Sequence[type] = (), ): - """Create a task specification class and its outputs specification class from the + """Create a task definition class and its outputs definition class from the input and output fields provided to the decorator/function. Modifies the class so that its attributes are converted from pydra fields to attrs fields @@ -380,16 +380,16 @@ def make_task_spec( name : str, optional The name of the class, by default bases : ty.Sequence[type], optional - The base classes for the task specification class, by default () + The base classes for the task definition class, by default () outputs_bases : ty.Sequence[type], optional - The base classes for the outputs specification class, by default () + The base classes for the outputs definition class, by default () Returns ------- klass : type The class created using the attrs package """ - from pydra.engine.specs import TaskSpec + from pydra.engine.specs import TaskDef spec_type._check_arg_refs(inputs, outputs) @@ -403,17 +403,17 @@ def make_task_spec( if klass is None or not issubclass(klass, spec_type): if name is None: raise ValueError("name must be provided if klass is not") - if klass is not None and issubclass(klass, TaskSpec): + if klass is not None and issubclass(klass, TaskDef): raise ValueError(f"Cannot change type of spec {klass} to {spec_type}") bases = tuple(bases) - # Ensure that TaskSpec is a base class + # Ensure that TaskDef is a base class if not any(issubclass(b, spec_type) for b in bases): bases = bases + (spec_type,) # If building from a decorated class (as opposed to dynamically from a function # or shell-template), add any base classes not already in the bases tuple if klass is not None: bases += tuple(c for c in klass.__mro__ if c not in bases + (object,)) - # Create a new class with the TaskSpec as a base class + # Create a new class with the TaskDef as a base class klass = types.new_class( name=name, bases=bases, @@ -472,7 +472,7 @@ def make_outputs_spec( bases: ty.Sequence[type], spec_name: str, ) -> type["TaskOutputs"]: - """Create an outputs specification class and its outputs specification class from the + """Create an outputs definition class and its outputs definition class from the output fields provided to the decorator/function. Creates a new class with attrs fields and then calls `attrs.define` to create an @@ -483,9 +483,9 @@ def make_outputs_spec( outputs : dict[str, Out] The output fields of the task bases : ty.Sequence[type], optional - The base classes for the outputs specification class, by default () + The base classes for the outputs definition class, by default () spec_name : str - The name of the task specification class the outputs are for + The name of the task definition class the outputs are for Returns ------- diff --git a/pydra/design/boutiques.py b/pydra/design/boutiques.py index 6877bf482..54050d60c 100644 --- a/pydra/design/boutiques.py +++ b/pydra/design/boutiques.py @@ -5,14 +5,14 @@ from pathlib import Path from functools import reduce from fileformats.generic import File -from pydra.engine.specs import ShellSpec +from pydra.engine.specs import ShellDef from pydra.engine.task import BoshTask from .base import make_task_spec from . import shell class arg(shell.arg): - """Class for input fields of Boutiques task specifications + """Class for input fields of Boutiques task definitions Parameters ---------- @@ -46,7 +46,7 @@ class arg(shell.arg): class out(shell.out): - """Class for output fields of Boutiques task specifications + """Class for output fields of Boutiques task definitions Parameters ---------- @@ -114,7 +114,7 @@ def define( bosh_spec, input_keys, names_subset=output_spec_names ) return make_task_spec( - spec_type=ShellSpec, + spec_type=ShellDef, task_type=BoshTask, out_type=out, arg_type=arg, diff --git a/pydra/design/python.py b/pydra/design/python.py index 625b88236..febc74b98 100644 --- a/pydra/design/python.py +++ b/pydra/design/python.py @@ -14,7 +14,7 @@ ) if ty.TYPE_CHECKING: - from pydra.engine.specs import PythonSpec + from pydra.engine.specs import PythonDef __all__ = ["arg", "out", "define"] @@ -101,7 +101,7 @@ def define( bases: ty.Sequence[type] = (), outputs_bases: ty.Sequence[type] = (), auto_attribs: bool = True, -) -> "PythonSpec": +) -> "PythonDef": """ Create an interface for a function or a class. @@ -118,13 +118,13 @@ def define( Returns ------- - PythonSpec - The task specification class for the Python function + PythonDef + The task definition class for the Python function """ from pydra.engine.task import PythonTask - from pydra.engine.specs import PythonSpec, PythonOutputs + from pydra.engine.specs import PythonDef, PythonOutputs - def make(wrapped: ty.Callable | type) -> PythonSpec: + def make(wrapped: ty.Callable | type) -> PythonDef: if inspect.isclass(wrapped): klass = wrapped function = klass.function @@ -160,7 +160,7 @@ def make(wrapped: ty.Callable | type) -> PythonSpec: ) interface = make_task_spec( - PythonSpec, + PythonDef, PythonOutputs, PythonTask, parsed_inputs, diff --git a/pydra/design/shell.py b/pydra/design/shell.py index 419279383..5e38f9ffa 100644 --- a/pydra/design/shell.py +++ b/pydra/design/shell.py @@ -25,7 +25,7 @@ from pydra.utils.typing import is_fileset_or_union, MultiInputObj if ty.TYPE_CHECKING: - from pydra.engine.specs import ShellSpec + from pydra.engine.specs import ShellDef __all__ = ["arg", "out", "outarg", "define"] @@ -190,7 +190,7 @@ class outarg(Out, arg): field will be sent). path_template: str, optional The template used to specify where the output file will be written to can use - other fields, e.g. {file1}. Used in order to create an output specification. + other fields, e.g. {file1}. Used in order to create an output definition. """ path_template: str | None = attrs.field(default=None) @@ -235,9 +235,9 @@ def define( outputs_bases: ty.Sequence[type] = (), auto_attribs: bool = True, name: str | None = None, -) -> "ShellSpec": - """Create a task specification for a shell command. Can be used either as a decorator on - the "canonical" dataclass-form of a task specification or as a function that takes a +) -> "ShellDef": + """Create a task definition for a shell command. Can be used either as a decorator on + the "canonical" dataclass-form of a task definition or as a function that takes a "shell-command template string" of the form ``` @@ -284,15 +284,15 @@ def define( Returns ------- - ShellSpec + ShellDef The interface for the shell command """ from pydra.engine.task import ShellTask - from pydra.engine.specs import ShellSpec, ShellOutputs + from pydra.engine.specs import ShellDef, ShellOutputs def make( wrapped: ty.Callable | type | None = None, - ) -> ShellSpec: + ) -> ShellDef: if inspect.isclass(wrapped): klass = wrapped @@ -374,7 +374,7 @@ def make( inpt.position = position_stack.pop(0) interface = make_task_spec( - ShellSpec, + ShellDef, ShellOutputs, ShellTask, parsed_inputs, @@ -684,5 +684,5 @@ class _InputPassThrough: name: str - def __call__(self, inputs: ShellSpec) -> ty.Any: + def __call__(self, inputs: ShellDef) -> ty.Any: return getattr(inputs, self.name) diff --git a/pydra/design/tests/test_python.py b/pydra/design/tests/test_python.py index 302c88d14..82c24c8fd 100644 --- a/pydra/design/tests/test_python.py +++ b/pydra/design/tests/test_python.py @@ -4,7 +4,7 @@ import attrs import pytest from pydra.engine.helpers import list_fields -from pydra.engine.specs import PythonSpec +from pydra.engine.specs import PythonDef from pydra.design import python from pydra.engine.task import PythonTask @@ -17,21 +17,21 @@ def func(a: int) -> float: """Sample function with inputs and outputs""" return a * 2 - SampleSpec = python.define(func) + SampleDef = python.define(func) - assert issubclass(SampleSpec, PythonSpec) - inputs = sorted(list_fields(SampleSpec), key=sort_key) - outputs = sorted(list_fields(SampleSpec.Outputs), key=sort_key) + assert issubclass(SampleDef, PythonDef) + inputs = sorted(list_fields(SampleDef), key=sort_key) + outputs = sorted(list_fields(SampleDef.Outputs), key=sort_key) assert inputs == [ python.arg(name="a", type=int), python.arg(name="function", type=ty.Callable, default=func), ] assert outputs == [python.out(name="out", type=float)] - spec = SampleSpec(a=1) + spec = SampleDef(a=1) result = spec() assert result.output.out == 2.0 with pytest.raises(TypeError): - SampleSpec(a=1.5) + SampleDef(a=1.5) def test_interface_wrap_function_with_default(): @@ -39,19 +39,19 @@ def func(a: int, k: float = 2.0) -> float: """Sample function with inputs and outputs""" return a * k - SampleSpec = python.define(func) + SampleDef = python.define(func) - assert issubclass(SampleSpec, PythonSpec) - inputs = sorted(list_fields(SampleSpec), key=sort_key) - outputs = sorted(list_fields(SampleSpec.Outputs), key=sort_key) + assert issubclass(SampleDef, PythonDef) + inputs = sorted(list_fields(SampleDef), key=sort_key) + outputs = sorted(list_fields(SampleDef.Outputs), key=sort_key) assert inputs == [ python.arg(name="a", type=int), python.arg(name="function", type=ty.Callable, default=func), python.arg(name="k", type=float, default=2.0), ] assert outputs == [python.out(name="out", type=float)] - assert SampleSpec(a=1)().output.out == 2.0 - assert SampleSpec(a=10, k=3.0)().output.out == 30.0 + assert SampleDef(a=1)().output.out == 2.0 + assert SampleDef(a=10, k=3.0)().output.out == 30.0 def test_interface_wrap_function_overrides(): @@ -59,15 +59,15 @@ def func(a: int) -> float: """Sample function with inputs and outputs""" return a * 2 - SampleSpec = python.define( + SampleDef = python.define( func, inputs={"a": python.arg(help_string="The argument to be doubled")}, outputs={"b": python.out(help_string="the doubled output", type=Decimal)}, ) - assert issubclass(SampleSpec, PythonSpec) - inputs = sorted(list_fields(SampleSpec), key=sort_key) - outputs = sorted(list_fields(SampleSpec.Outputs), key=sort_key) + assert issubclass(SampleDef, PythonDef) + inputs = sorted(list_fields(SampleDef), key=sort_key) + outputs = sorted(list_fields(SampleDef.Outputs), key=sort_key) assert inputs == [ python.arg(name="a", type=int, help_string="The argument to be doubled"), python.arg(name="function", type=ty.Callable, default=func), @@ -75,7 +75,7 @@ def func(a: int) -> float: assert outputs == [ python.out(name="b", type=Decimal, help_string="the doubled output"), ] - outputs = SampleSpec.Outputs(b=Decimal(2.0)) + outputs = SampleDef.Outputs(b=Decimal(2.0)) assert isinstance(outputs.b, Decimal) @@ -84,84 +84,84 @@ def func(a: int) -> int: """Sample function with inputs and outputs""" return a * 2 - SampleSpec = python.define( + SampleDef = python.define( func, inputs={"a": float}, outputs={"b": float}, ) - assert issubclass(SampleSpec, PythonSpec) - inputs = sorted(list_fields(SampleSpec), key=sort_key) - outputs = sorted(list_fields(SampleSpec.Outputs), key=sort_key) + assert issubclass(SampleDef, PythonDef) + inputs = sorted(list_fields(SampleDef), key=sort_key) + outputs = sorted(list_fields(SampleDef.Outputs), key=sort_key) assert inputs == [ python.arg(name="a", type=float), python.arg(name="function", type=ty.Callable, default=func), ] assert outputs == [python.out(name="b", type=float)] - intf = SampleSpec(a=1) + intf = SampleDef(a=1) assert isinstance(intf.a, float) - outputs = SampleSpec.Outputs(b=2.0) + outputs = SampleDef.Outputs(b=2.0) assert isinstance(outputs.b, float) def test_decorated_function_interface(): @python.define(outputs=["c", "d"]) - def SampleSpec(a: int, b: float) -> tuple[float, float]: + def SampleDef(a: int, b: float) -> tuple[float, float]: """Sample function for testing""" return a + b, a * b - assert issubclass(SampleSpec, PythonSpec) - assert SampleSpec.Task is PythonTask - inputs = sorted(list_fields(SampleSpec), key=sort_key) - outputs = sorted(list_fields(SampleSpec.Outputs), key=sort_key) + assert issubclass(SampleDef, PythonDef) + assert SampleDef.Task is PythonTask + inputs = sorted(list_fields(SampleDef), key=sort_key) + outputs = sorted(list_fields(SampleDef.Outputs), key=sort_key) assert inputs == [ python.arg(name="a", type=int), python.arg(name="b", type=float), python.arg( name="function", type=ty.Callable, - default=attrs.fields(SampleSpec).function.default, + default=attrs.fields(SampleDef).function.default, ), ] assert outputs == [ python.out(name="c", type=float), python.out(name="d", type=float), ] - assert attrs.fields(SampleSpec).function.default.__name__ == "SampleSpec" - SampleSpec.Outputs(c=1.0, d=2.0) + assert attrs.fields(SampleDef).function.default.__name__ == "SampleDef" + SampleDef.Outputs(c=1.0, d=2.0) def test_interface_with_function_implicit_outputs_from_return_stmt(): @python.define - def SampleSpec(a: int, b: float) -> tuple[float, float]: + def SampleDef(a: int, b: float) -> tuple[float, float]: """Sample function for testing""" c = a + b d = a * b return c, d - assert SampleSpec.Task is PythonTask - inputs = sorted(list_fields(SampleSpec), key=sort_key) - outputs = sorted(list_fields(SampleSpec.Outputs), key=sort_key) + assert SampleDef.Task is PythonTask + inputs = sorted(list_fields(SampleDef), key=sort_key) + outputs = sorted(list_fields(SampleDef.Outputs), key=sort_key) assert inputs == [ python.arg(name="a", type=int), python.arg(name="b", type=float), python.arg( name="function", type=ty.Callable, - default=attrs.fields(SampleSpec).function.default, + default=attrs.fields(SampleDef).function.default, ), ] assert outputs == [ python.out(name="c", type=float), python.out(name="d", type=float), ] - assert attrs.fields(SampleSpec).function.default.__name__ == "SampleSpec" - SampleSpec.Outputs(c=1.0, d=2.0) + assert attrs.fields(SampleDef).function.default.__name__ == "SampleDef" + SampleDef.Outputs(c=1.0, d=2.0) def test_interface_with_function_docstr(): @python.define(outputs=["c", "d"]) - def SampleSpec(a: int, b: float) -> tuple[float, float]: + def SampleDef(a: int, b: float) -> tuple[float, float]: """Sample function for testing :param a: First input to be inputted @@ -171,28 +171,28 @@ def SampleSpec(a: int, b: float) -> tuple[float, float]: """ return a + b, a * b - assert SampleSpec.Task is PythonTask - inputs = sorted(list_fields(SampleSpec), key=sort_key) - outputs = sorted(list_fields(SampleSpec.Outputs), key=sort_key) + assert SampleDef.Task is PythonTask + inputs = sorted(list_fields(SampleDef), key=sort_key) + outputs = sorted(list_fields(SampleDef.Outputs), key=sort_key) assert inputs == [ python.arg(name="a", type=int, help_string="First input to be inputted"), python.arg(name="b", type=float, help_string="Second input"), python.arg( name="function", type=ty.Callable, - default=attrs.fields(SampleSpec).function.default, + default=attrs.fields(SampleDef).function.default, ), ] assert outputs == [ python.out(name="c", type=float, help_string="Sum of a and b"), python.out(name="d", type=float, help_string="product of a and b"), ] - assert attrs.fields(SampleSpec).function.default.__name__ == "SampleSpec" + assert attrs.fields(SampleDef).function.default.__name__ == "SampleDef" def test_interface_with_function_google_docstr(): @python.define(outputs=["c", "d"]) - def SampleSpec(a: int, b: float) -> tuple[float, float]: + def SampleDef(a: int, b: float) -> tuple[float, float]: """Sample function for testing Args: @@ -206,30 +206,30 @@ def SampleSpec(a: int, b: float) -> tuple[float, float]: """ return a + b, a * b - assert SampleSpec.Task is PythonTask - inputs = sorted(list_fields(SampleSpec), key=sort_key) - outputs = sorted(list_fields(SampleSpec.Outputs), key=sort_key) + assert SampleDef.Task is PythonTask + inputs = sorted(list_fields(SampleDef), key=sort_key) + outputs = sorted(list_fields(SampleDef.Outputs), key=sort_key) assert inputs == [ python.arg(name="a", type=int, help_string="First input to be inputted"), python.arg(name="b", type=float, help_string="Second input"), python.arg( name="function", type=ty.Callable, - default=attrs.fields(SampleSpec).function.default, + default=attrs.fields(SampleDef).function.default, ), ] assert outputs == [ python.out(name="c", type=float, help_string="Sum of a and b"), python.out(name="d", type=float, help_string="Product of a and b"), ] - assert attrs.fields(SampleSpec).function.default.__name__ == "SampleSpec" + assert attrs.fields(SampleDef).function.default.__name__ == "SampleDef" def test_interface_with_function_numpy_docstr(): @python.define( outputs=["c", "d"] ) # Could potentiall read output names from doc-string instead - def SampleSpec(a: int, b: float) -> tuple[float, float]: + def SampleDef(a: int, b: float) -> tuple[float, float]: """Sample function for testing Parameters @@ -249,28 +249,28 @@ def SampleSpec(a: int, b: float) -> tuple[float, float]: """ return a + b, a * b - assert SampleSpec.Task is PythonTask - inputs = sorted(list_fields(SampleSpec), key=sort_key) - outputs = sorted(list_fields(SampleSpec.Outputs), key=sort_key) + assert SampleDef.Task is PythonTask + inputs = sorted(list_fields(SampleDef), key=sort_key) + outputs = sorted(list_fields(SampleDef.Outputs), key=sort_key) assert inputs == [ python.arg(name="a", type=int, help_string="First input to be inputted"), python.arg(name="b", type=float, help_string="Second input"), python.arg( name="function", type=ty.Callable, - default=attrs.fields(SampleSpec).function.default, + default=attrs.fields(SampleDef).function.default, ), ] assert outputs == [ python.out(name="c", type=float, help_string="Sum of a and b"), python.out(name="d", type=float, help_string="Product of a and b"), ] - assert attrs.fields(SampleSpec).function.default.__name__ == "SampleSpec" + assert attrs.fields(SampleDef).function.default.__name__ == "SampleDef" def test_interface_with_class(): @python.define - class SampleSpec: + class SampleDef: """Sample class for testing Args: @@ -296,32 +296,32 @@ class Outputs: def function(a, b): return a + b, a * b - assert issubclass(SampleSpec, PythonSpec) - assert SampleSpec.Task is PythonTask - inputs = sorted(list_fields(SampleSpec), key=sort_key) - outputs = sorted(list_fields(SampleSpec.Outputs), key=sort_key) + assert issubclass(SampleDef, PythonDef) + assert SampleDef.Task is PythonTask + inputs = sorted(list_fields(SampleDef), key=sort_key) + outputs = sorted(list_fields(SampleDef.Outputs), key=sort_key) assert inputs == [ python.arg(name="a", type=int, help_string="First input to be inputted"), python.arg(name="b", type=float, default=2.0, help_string="Second input"), python.arg( name="function", type=ty.Callable, - default=attrs.fields(SampleSpec).function.default, + default=attrs.fields(SampleDef).function.default, ), ] assert outputs == [ python.out(name="c", type=float, help_string="Sum of a and b"), python.out(name="d", type=float, help_string="Product of a and b"), ] - assert SampleSpec.function.__name__ == "function" - SampleSpec(a=1) - SampleSpec(a=1, b=2.0) - SampleSpec.Outputs(c=1.0, d=2.0) + assert SampleDef.function.__name__ == "function" + SampleDef(a=1) + SampleDef(a=1, b=2.0) + SampleDef.Outputs(c=1.0, d=2.0) def test_interface_with_inheritance(): @python.define - class SampleSpec(PythonSpec["SampleSpec.Outputs"]): + class SampleDef(PythonDef["SampleDef.Outputs"]): """Sample class for testing Args: @@ -347,12 +347,12 @@ class Outputs: def function(a, b): return a + b, a * b - assert issubclass(SampleSpec, PythonSpec) + assert issubclass(SampleDef, PythonDef) def test_interface_with_class_no_auto_attribs(): @python.define(auto_attribs=False) - class SampleSpec: + class SampleDef: a: int = python.arg(help_string="First input to be inputted") b: float = python.arg(help_string="Second input") @@ -368,36 +368,36 @@ class Outputs: def function(a, b): return a + b, a * b - assert SampleSpec.Task is PythonTask - inputs = sorted(list_fields(SampleSpec), key=sort_key) - outputs = sorted(list_fields(SampleSpec.Outputs), key=sort_key) + assert SampleDef.Task is PythonTask + inputs = sorted(list_fields(SampleDef), key=sort_key) + outputs = sorted(list_fields(SampleDef.Outputs), key=sort_key) assert inputs == [ python.arg(name="a", type=int, help_string="First input to be inputted"), python.arg(name="b", type=float, help_string="Second input"), python.arg( name="function", type=ty.Callable, - default=attrs.fields(SampleSpec).function.default, + default=attrs.fields(SampleDef).function.default, ), ] assert outputs == [ python.out(name="c", type=float, help_string="Sum of a and b"), python.out(name="d", type=float, help_string="Product of a and b"), ] - assert SampleSpec.function.__name__ == "function" - SampleSpec(a=1, b=2.0) - SampleSpec.Outputs(c=1.0, d=2.0) + assert SampleDef.function.__name__ == "function" + SampleDef(a=1, b=2.0) + SampleDef.Outputs(c=1.0, d=2.0) with pytest.raises(TypeError): - SampleSpec(a=1, b=2.0, x=3) + SampleDef(a=1, b=2.0, x=3) with pytest.raises(TypeError): - SampleSpec.Outputs(c=1.0, d=2.0, y="hello") + SampleDef.Outputs(c=1.0, d=2.0, y="hello") def test_interface_invalid_wrapped1(): with pytest.raises(ValueError): @python.define(inputs={"a": python.arg()}) - class SampleSpec(PythonSpec["SampleSpec.Outputs"]): + class SampleDef(PythonDef["SampleDef.Outputs"]): a: int class Outputs: @@ -412,7 +412,7 @@ def test_interface_invalid_wrapped2(): with pytest.raises(ValueError): @python.define(outputs={"b": python.out()}) - class SampleSpec(PythonSpec["SampleSpec.Outputs"]): + class SampleDef(PythonDef["SampleDef.Outputs"]): a: int class Outputs: diff --git a/pydra/design/tests/test_shell.py b/pydra/design/tests/test_shell.py index 771bcba99..1f35b4e82 100644 --- a/pydra/design/tests/test_shell.py +++ b/pydra/design/tests/test_shell.py @@ -7,7 +7,7 @@ from pydra.design import shell from pydra.engine.helpers import list_fields from pydra.engine.specs import ( - ShellSpec, + ShellDef, ShellOutputs, RETURN_CODE_HELP, STDOUT_HELP, @@ -22,7 +22,7 @@ def test_interface_template(): Cp = shell.define("cp ") - assert issubclass(Cp, ShellSpec) + assert issubclass(Cp, ShellDef) output = shell.outarg( name="out_path", path_template="out_path", @@ -69,7 +69,7 @@ def test_interface_template_w_types_and_path_template_ext(): TrimPng = shell.define("trim-png ") - assert issubclass(TrimPng, ShellSpec) + assert issubclass(TrimPng, ShellDef) output = shell.outarg( name="out_image", path_template="out_image.png", @@ -115,7 +115,7 @@ def test_interface_template_w_modify(): TrimPng = shell.define("trim-png ") - assert issubclass(TrimPng, ShellSpec) + assert issubclass(TrimPng, ShellDef) assert sorted_fields(TrimPng) == [ shell.arg( name="executable", @@ -167,7 +167,7 @@ def test_interface_template_more_complex(): ), ) - assert issubclass(Cp, ShellSpec) + assert issubclass(Cp, ShellDef) output = shell.outarg( name="out_dir", type=Directory, @@ -254,7 +254,7 @@ def test_interface_template_with_overrides_and_optionals(): }, ) - assert issubclass(Cp, ShellSpec) + assert issubclass(Cp, ShellDef) outargs = [ shell.outarg( name="out_dir", @@ -340,7 +340,7 @@ def test_interface_template_with_defaults(): ), ) - assert issubclass(Cp, ShellSpec) + assert issubclass(Cp, ShellDef) output = shell.outarg( name="out_dir", type=Directory, @@ -408,7 +408,7 @@ def test_interface_template_with_type_overrides(): inputs={"text_arg": str, "int_arg": int | None}, ) - assert issubclass(Cp, ShellSpec) + assert issubclass(Cp, ShellDef) output = shell.outarg( name="out_dir", type=Directory, @@ -468,7 +468,7 @@ def Ls(request): if request.param == "static": @shell.define - class Ls(ShellSpec["Ls.Outputs"]): + class Ls(ShellDef["Ls.Outputs"]): executable = "ls" directory: Directory = shell.arg( diff --git a/pydra/design/tests/test_workflow.py b/pydra/design/tests/test_workflow.py index d6b11ab56..10d091db3 100644 --- a/pydra/design/tests/test_workflow.py +++ b/pydra/design/tests/test_workflow.py @@ -7,7 +7,7 @@ import typing as ty from pydra.design import shell, python, workflow from pydra.engine.helpers import list_fields -from pydra.engine.specs import WorkflowSpec, WorkflowOutputs +from pydra.engine.specs import WorkflowDef, WorkflowOutputs from fileformats import video, image # NB: We use PascalCase for interfaces and workflow functions as it is translated into a class @@ -50,7 +50,7 @@ def MyTestWorkflow(a, b): constructor = MyTestWorkflow().constructor assert constructor.__name__ == "MyTestWorkflow" - # The constructor function is included as a part of the specification so it is + # The constructor function is included as a part of the definition so it is # included in the hash by default and can be overridden if needed. Not 100% sure # if this is a good idea or not assert list_fields(MyTestWorkflow) == [ @@ -133,7 +133,7 @@ def test_workflow_canonical(): # NB: We use PascalCase (i.e. class names) as it is translated into a class @workflow.define - class MyTestWorkflow(WorkflowSpec["MyTestWorkflow.Outputs"]): + class MyTestWorkflow(WorkflowDef["MyTestWorkflow.Outputs"]): a: int b: float = workflow.arg( @@ -154,7 +154,7 @@ class Outputs(WorkflowOutputs): constructor = MyTestWorkflow().constructor assert constructor.__name__ == "constructor" - # The constructor function is included as a part of the specification so it is + # The constructor function is included as a part of the definition so it is # included in the hash by default and can be overridden if needed. Not 100% sure # if this is a good idea or not assert sorted(list_fields(MyTestWorkflow), key=attrgetter("name")) == [ diff --git a/pydra/design/workflow.py b/pydra/design/workflow.py index 7562d78df..32c84d5a6 100644 --- a/pydra/design/workflow.py +++ b/pydra/design/workflow.py @@ -15,7 +15,7 @@ if ty.TYPE_CHECKING: from pydra.engine.workflow.base import Workflow - from pydra.engine.specs import TaskSpec, TaskOutputs, WorkflowSpec + from pydra.engine.specs import TaskDef, TaskOutputs, WorkflowDef __all__ = ["define", "add", "this", "arg", "out"] @@ -107,10 +107,10 @@ def define( outputs_bases: ty.Sequence[type] = (), lazy: list[str] | None = None, auto_attribs: bool = True, -) -> "WorkflowSpec": +) -> "WorkflowDef": """ Create an interface for a function or a class. Can be used either as a decorator on - a constructor function or the "canonical" dataclass-form of a task specification. + a constructor function or the "canonical" dataclass-form of a task definition. Parameters ---------- @@ -125,16 +125,16 @@ def define( Returns ------- - TaskSpec + TaskDef The interface for the function or class. """ from pydra.engine.core import WorkflowTask - from pydra.engine.specs import TaskSpec, WorkflowSpec, WorkflowOutputs + from pydra.engine.specs import TaskDef, WorkflowDef, WorkflowOutputs if lazy is None: lazy = [] - def make(wrapped: ty.Callable | type) -> TaskSpec: + def make(wrapped: ty.Callable | type) -> TaskDef: if inspect.isclass(wrapped): klass = wrapped constructor = klass.constructor @@ -172,7 +172,7 @@ def make(wrapped: ty.Callable | type) -> TaskSpec: parsed_inputs[inpt_name].lazy = True interface = make_task_spec( - WorkflowSpec, + WorkflowDef, WorkflowOutputs, WorkflowTask, parsed_inputs, @@ -208,20 +208,20 @@ def this() -> "Workflow": OutputsType = ty.TypeVar("OutputsType", bound="TaskOutputs") -def add(task_spec: "TaskSpec[OutputsType]", name: str = None) -> OutputsType: +def add(task_spec: "TaskDef[OutputsType]", name: str = None) -> OutputsType: """Add a node to the workflow currently being constructed Parameters ---------- - task_spec : TaskSpec - The specification of the task to add to the workflow as a node + task_spec : TaskDef + The definition of the task to add to the workflow as a node name : str, optional - The name of the node, by default it will be the name of the task specification + The name of the node, by default it will be the name of the task definition class Returns ------- Outputs - The outputs specification of the node + The outputs definition of the node """ return this().add(task_spec, name=name) diff --git a/pydra/engine/audit.py b/pydra/engine/audit.py index 6f8d2dd8c..6f39fda1d 100644 --- a/pydra/engine/audit.py +++ b/pydra/engine/audit.py @@ -26,7 +26,7 @@ def __init__(self, audit_flags, messengers, messenger_args, develop=None): Base configuration of auditing. messengers : :obj:`pydra.util.messenger.Messenger` or list of :class:`pydra.util.messenger.Messenger`, optional - Specify types of messenger used by Audit to send a message. + Defify types of messenger used by Audit to send a message. Could be `PrintMessenger`, `FileMessenger`, or `RemoteRESTMessenger`. messenger_args : :obj:`dict`, optional Optional arguments for the `Messenger.send` method. diff --git a/pydra/engine/core.py b/pydra/engine/core.py index 780a2d997..904bbf9b9 100644 --- a/pydra/engine/core.py +++ b/pydra/engine/core.py @@ -19,10 +19,10 @@ from . import helpers_state as hlpst from .specs import ( File, - RuntimeSpec, + RuntimeDef, Result, TaskHook, - TaskSpec, + TaskDef, ) from .helpers import ( create_checksum, @@ -71,14 +71,14 @@ class Task: _can_resume = False # Does the task allow resuming from previous state _redirect_x = False # Whether an X session should be created/directed - _runtime_requirements = RuntimeSpec() + _runtime_requirements = RuntimeDef() _runtime_hints = None _cache_dir = None # Working directory in which to operate _references = None # List of references for a task name: str - spec: TaskSpec + spec: TaskDef _inputs: dict[str, ty.Any] | None = None @@ -336,7 +336,7 @@ def __call__( from .submitter import Submitter if submitter and plugin: - raise Exception("Specify submitter OR plugin, not both") + raise Exception("Defify submitter OR plugin, not both") elif submitter: pass # if there is plugin provided or the task is a Workflow or has a state, @@ -1033,7 +1033,7 @@ async def _run_task(self, submitter, rerun=False, environment=None): # else: # type_ = lf.type # fields.append((wf_out_nm, type_, {"help_string": help_string})) - # self.output_spec = SpecInfo(name="Output", fields=fields, bases=(BaseSpec,)) + # self.output_spec = SpecInfo(name="Output", fields=fields, bases=(BaseDef,)) # logger.info("Added %s to %s", self.output_spec, self) def _collect_outputs(self): diff --git a/pydra/engine/helpers.py b/pydra/engine/helpers.py index 8b2cc6428..edf292ed9 100644 --- a/pydra/engine/helpers.py +++ b/pydra/engine/helpers.py @@ -18,7 +18,7 @@ from fileformats.core import FileSet if ty.TYPE_CHECKING: - from .specs import TaskSpec + from .specs import TaskDef from pydra.design.base import Field @@ -35,8 +35,8 @@ def attrs_values(obj, **kwargs) -> dict[str, ty.Any]: return attrs.asdict(obj, recurse=False, **kwargs) -def list_fields(spec: "type[TaskSpec] | TaskSpec") -> list["Field"]: - """List the fields of a task specification""" +def list_fields(spec: "type[TaskDef] | TaskDef") -> list["Field"]: + """List the fields of a task definition""" if not inspect.isclass(spec): spec = type(spec) if not attrs.has(spec): @@ -48,7 +48,7 @@ def list_fields(spec: "type[TaskSpec] | TaskSpec") -> list["Field"]: ] -def fields_dict(spec: "type[TaskSpec] | TaskSpec") -> dict[str, "Field"]: +def fields_dict(spec: "type[TaskDef] | TaskDef") -> dict[str, "Field"]: """Returns the fields of a spec in a dictionary""" return {f.name: f for f in list_fields(spec)} diff --git a/pydra/engine/specs.py b/pydra/engine/specs.py index 8dcb196a3..a30861ca3 100644 --- a/pydra/engine/specs.py +++ b/pydra/engine/specs.py @@ -1,4 +1,4 @@ -"""Task I/O specifications.""" +"""Task I/O definitions.""" from pathlib import Path import re @@ -41,7 +41,7 @@ def is_set(value: ty.Any) -> bool: class TaskOutputs: - """Base class for all output specifications""" + """Base class for all output definitions""" RESERVED_FIELD_NAMES = ("inputs", "split", "combine") @@ -145,7 +145,7 @@ def _get_node(self): ) def __iter__(self) -> ty.Generator[str, None, None]: - """Iterate through all the names in the specification""" + """Iterate through all the names in the definition""" return (f.name for f in list_fields(self)) def __getitem__(self, name: str) -> ty.Any: @@ -170,8 +170,8 @@ def __getitem__(self, name: str) -> ty.Any: OutputsType = ty.TypeVar("OutputType", bound=TaskOutputs) -class TaskSpec(ty.Generic[OutputsType]): - """Base class for all task specifications""" +class TaskDef(ty.Generic[OutputsType]): + """Base class for all task definitions""" Task: "ty.Type[core.Task]" @@ -190,7 +190,7 @@ def __call__( rerun=False, **kwargs, ) -> "Result[OutputsType]": - """Create a task from this specification and execute it to produce a result. + """Create a task from this definition and execute it to produce a result. Parameters ---------- @@ -236,7 +236,7 @@ def __call__( return task(**kwargs) def __iter__(self) -> ty.Generator[str, None, None]: - """Iterate through all the names in the specification""" + """Iterate through all the names in the definition""" return (f.name for f in list_fields(self)) def __getitem__(self, name: str) -> ty.Any: @@ -429,7 +429,7 @@ def get_output_field(self, field_name): @attrs.define(kw_only=True) -class RuntimeSpec: +class RuntimeDef: """ Specification for a task. @@ -460,7 +460,7 @@ class PythonOutputs(TaskOutputs): PythonOutputsType = ty.TypeVar("OutputType", bound=PythonOutputs) -class PythonSpec(TaskSpec[PythonOutputsType]): +class PythonDef(TaskDef[PythonOutputsType]): pass @@ -471,7 +471,7 @@ class WorkflowOutputs(TaskOutputs): WorkflowOutputsType = ty.TypeVar("OutputType", bound=WorkflowOutputs) -class WorkflowSpec(TaskSpec[WorkflowOutputsType]): +class WorkflowDef(TaskDef[WorkflowOutputsType]): pass @@ -481,7 +481,7 @@ class WorkflowSpec(TaskSpec[WorkflowOutputsType]): class ShellOutputs(TaskOutputs): - """Output specification of a generic shell process.""" + """Output definition of a generic shell process.""" return_code: int = shell.out(help_string=RETURN_CODE_HELP) stdout: str = shell.out(help_string=STDOUT_HELP) @@ -497,8 +497,8 @@ def from_task( Parameters ---------- - inputs : ShellSpec - The input specification of the shell process. + inputs : ShellDef + The input definition of the shell process. output_dir : Path The directory where the process was run. stdout : str @@ -556,7 +556,7 @@ def _resolve_default_value(cls, fld: shell.out, output_dir: Path) -> ty.Any: return default @classmethod - def _required_fields_satisfied(cls, fld: shell.out, inputs: "ShellSpec") -> bool: + def _required_fields_satisfied(cls, fld: shell.out, inputs: "ShellDef") -> bool: """checking if all fields from the requires and template are set in the input if requires is a list of list, checking if at least one list has all elements set """ @@ -587,7 +587,7 @@ def _required_fields_satisfied(cls, fld: shell.out, inputs: "ShellSpec") -> bool ShellOutputsType = ty.TypeVar("OutputType", bound=ShellOutputs) -class ShellSpec(TaskSpec[ShellOutputsType]): +class ShellDef(TaskDef[ShellOutputsType]): RESERVED_FIELD_NAMES = ("cmdline",) diff --git a/pydra/engine/state.py b/pydra/engine/state.py index fb26767e5..58a1a68b0 100644 --- a/pydra/engine/state.py +++ b/pydra/engine/state.py @@ -7,7 +7,7 @@ from . import helpers_state as hlpst from .helpers import ensure_list, attrs_values -# from .specs import BaseSpec +# from .specs import BaseDef # TODO: move to State op = {".": zip, "*": itertools.product} diff --git a/pydra/engine/task.py b/pydra/engine/task.py index fe9816992..b3377a846 100644 --- a/pydra/engine/task.py +++ b/pydra/engine/task.py @@ -50,8 +50,8 @@ from .core import Task from pydra.utils.messenger import AuditFlag from .specs import ( - PythonSpec, - ShellSpec, + PythonDef, + ShellDef, is_set, attrs_fields, ) @@ -70,7 +70,7 @@ class PythonTask(Task): """Wrap a Python callable as a task element.""" - spec: PythonSpec + spec: PythonDef def _run_task(self, environment=None): inputs = attrs_values(self.spec) @@ -97,11 +97,11 @@ def _run_task(self, environment=None): class ShellTask(Task): """Wrap a shell command as a task element.""" - spec: ShellSpec + spec: ShellDef def __init__( self, - spec: ShellSpec, + spec: ShellDef, audit_flags: AuditFlag = AuditFlag.NONE, cache_dir=None, cont_dim=None, @@ -133,7 +133,7 @@ def __init__( TODO name : :obj:`str` Name of this task. - output_spec : :obj:`pydra.engine.specs.BaseSpec` + output_spec : :obj:`pydra.engine.specs.BaseDef` Specification of inputs. strip : :obj:`bool` TODO diff --git a/pydra/engine/tests/test_helpers_file.py b/pydra/engine/tests/test_helpers_file.py index 838e054bd..c0973379c 100644 --- a/pydra/engine/tests/test_helpers_file.py +++ b/pydra/engine/tests/test_helpers_file.py @@ -5,7 +5,7 @@ from unittest.mock import Mock import pytest from fileformats.generic import File -from ..specs import ShellSpec +from ..specs import ShellDef from ..task import ShellTask from ..helpers_file import ( ensure_list, @@ -385,7 +385,7 @@ def test_output_template(tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) class MyCommand(ShellTask): diff --git a/pydra/engine/tests/test_nipype1_convert.py b/pydra/engine/tests/test_nipype1_convert.py index 17384b764..a93492a0f 100644 --- a/pydra/engine/tests/test_nipype1_convert.py +++ b/pydra/engine/tests/test_nipype1_convert.py @@ -1,7 +1,7 @@ import typing as ty import pytest from pathlib import Path -from pydra.engine.specs import ShellOutputs, ShellSpec +from pydra.engine.specs import ShellOutputs, ShellDef from fileformats.generic import File from pydra.design import shell @@ -21,7 +21,7 @@ def find_txt(output_dir: Path) -> File: @shell.define -class Interf_3(ShellSpec["Interf_3.Outputs"]): +class Interf_3(ShellDef["Interf_3.Outputs"]): """class with customized input and executables""" executable = ["testing", "command"] @@ -34,7 +34,7 @@ class Outputs(ShellOutputs): @shell.define -class TouchInterf(ShellSpec["TouchInterf.Outputs"]): +class TouchInterf(ShellDef["TouchInterf.Outputs"]): """class with customized input and executables""" new_file: str = shell.outarg( @@ -58,7 +58,7 @@ def test_interface_specs_2(): my_input_spec = SpecInfo( name="Input", fields=[("my_inp", ty.Any, {"help_string": "my inp"})], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_output_spec = SpecInfo( name="Output", fields=[("my_out", File, "*.txt")], bases=(ShellOutputs,) diff --git a/pydra/engine/tests/test_shelltask.py b/pydra/engine/tests/test_shelltask.py index 9b70dbc6b..d03dbd466 100644 --- a/pydra/engine/tests/test_shelltask.py +++ b/pydra/engine/tests/test_shelltask.py @@ -11,7 +11,7 @@ from ..submitter import Submitter from ..specs import ( ShellOutputs, - ShellSpec, + ShellDef, ) from fileformats.generic import ( File, @@ -296,7 +296,7 @@ def test_shell_cmd_inputspec_1(plugin, results_function, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -344,7 +344,7 @@ def test_shell_cmd_inputspec_2(plugin, results_function, tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -385,7 +385,7 @@ def test_shell_cmd_inputspec_3(plugin, results_function, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -418,7 +418,7 @@ def test_shell_cmd_inputspec_3a(plugin, results_function, tmp_path): {"position": 1, "help_string": "text", "mandatory": True, "argstr": ""}, ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -456,7 +456,7 @@ def test_shell_cmd_inputspec_3b(plugin, results_function, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -490,7 +490,7 @@ def test_shell_cmd_inputspec_3c_exception(plugin, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -523,7 +523,7 @@ def test_shell_cmd_inputspec_3c(plugin, results_function, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -553,7 +553,7 @@ def test_shell_cmd_inputspec_4(plugin, results_function, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -579,7 +579,7 @@ def test_shell_cmd_inputspec_4a(plugin, results_function, tmp_path): fields=[ ("text", str, "Hello", {"position": 1, "help_string": "text", "argstr": ""}) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -610,7 +610,7 @@ def test_shell_cmd_inputspec_4b(plugin, results_function, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -645,7 +645,7 @@ def test_shell_cmd_inputspec_4c_exception(plugin): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -675,7 +675,7 @@ def test_shell_cmd_inputspec_4d_exception(plugin): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -718,7 +718,7 @@ def test_shell_cmd_inputspec_5_nosubm(plugin, results_function, tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -767,7 +767,7 @@ def test_shell_cmd_inputspec_5a_exception(plugin, tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -814,7 +814,7 @@ def test_shell_cmd_inputspec_6(plugin, results_function, tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -860,7 +860,7 @@ def test_shell_cmd_inputspec_6a_exception(plugin): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -902,7 +902,7 @@ def test_shell_cmd_inputspec_6b(plugin, results_function, tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -943,7 +943,7 @@ def test_shell_cmd_inputspec_7(plugin, results_function, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -988,7 +988,7 @@ def test_shell_cmd_inputspec_7a(plugin, results_function, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -1035,7 +1035,7 @@ def test_shell_cmd_inputspec_7b(plugin, results_function, tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -1074,7 +1074,7 @@ def test_shell_cmd_inputspec_7c(plugin, results_function, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -1132,7 +1132,7 @@ def test_shell_cmd_inputspec_8(plugin, results_function, tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -1189,7 +1189,7 @@ def test_shell_cmd_inputspec_8a(plugin, results_function, tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -1240,7 +1240,7 @@ def test_shell_cmd_inputspec_9(tmp_path, plugin, results_function): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -1293,7 +1293,7 @@ def test_shell_cmd_inputspec_9a(tmp_path, plugin, results_function): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -1341,7 +1341,7 @@ def test_shell_cmd_inputspec_9b(tmp_path, plugin, results_function): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -1392,7 +1392,7 @@ def test_shell_cmd_inputspec_9c(tmp_path, plugin, results_function): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -1444,7 +1444,7 @@ def test_shell_cmd_inputspec_9d(tmp_path, plugin, results_function): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -1495,7 +1495,7 @@ def test_shell_cmd_inputspec_10(plugin, results_function, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -1543,7 +1543,7 @@ def test_shell_cmd_inputspec_10_err(tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) with pytest.raises(FileNotFoundError): @@ -1579,7 +1579,7 @@ def test_shell_cmd_inputspec_11(tmp_path): ) ] - input_spec = SpecInfo(name="Input", fields=input_fields, bases=(ShellSpec,)) + input_spec = SpecInfo(name="Input", fields=input_fields, bases=(ShellDef,)) output_spec = SpecInfo(name="Output", fields=output_fields, bases=(ShellOutputs,)) task = ShellTask( @@ -1655,7 +1655,7 @@ def template_function(inputs): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -1699,7 +1699,7 @@ def test_shell_cmd_inputspec_with_iterable(): }, ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) task = ShellTask(name="test", input_spec=input_spec, executable="test") @@ -1749,7 +1749,7 @@ def test_shell_cmd_inputspec_copyfile_1(plugin, results_function, tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -1811,7 +1811,7 @@ def test_shell_cmd_inputspec_copyfile_1a(plugin, results_function, tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -1889,7 +1889,7 @@ def test_shell_cmd_inputspec_copyfile_1b(plugin, results_function, tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -1930,7 +1930,7 @@ def test_shell_cmd_inputspec_state_1(plugin, results_function, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -1965,7 +1965,7 @@ def test_shell_cmd_inputspec_typeval_1(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) with pytest.raises(TypeError): @@ -1981,7 +1981,7 @@ def test_shell_cmd_inputspec_typeval_2(): my_input_spec = SpecInfo( name="Input", fields=[("text", int, {"position": 1, "argstr": "", "help_string": "text"})], - bases=(ShellSpec,), + bases=(ShellDef,), ) with pytest.raises(TypeError): @@ -2003,7 +2003,7 @@ def test_shell_cmd_inputspec_state_1a(plugin, results_function, tmp_path): {"position": 1, "help_string": "text", "mandatory": True, "argstr": ""}, ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -2042,7 +2042,7 @@ def test_shell_cmd_inputspec_state_2(plugin, results_function, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -2088,7 +2088,7 @@ def test_shell_cmd_inputspec_state_3(plugin, results_function, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -2148,7 +2148,7 @@ def test_shell_cmd_inputspec_copyfile_state_1(plugin, results_function, tmp_path ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -2200,7 +2200,7 @@ def test_wf_shell_cmd_2(plugin_dask_opt, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) wf.add( @@ -2247,7 +2247,7 @@ def test_wf_shell_cmd_2a(plugin, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) wf.add( @@ -2295,7 +2295,7 @@ def test_wf_shell_cmd_3(plugin, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_input_spec2 = SpecInfo( @@ -2325,7 +2325,7 @@ def test_wf_shell_cmd_3(plugin, tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) wf.add( @@ -2392,7 +2392,7 @@ def test_wf_shell_cmd_3a(plugin, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_input_spec2 = SpecInfo( @@ -2422,7 +2422,7 @@ def test_wf_shell_cmd_3a(plugin, tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) wf.add( @@ -2487,7 +2487,7 @@ def test_wf_shell_cmd_state_1(plugin, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_input_spec2 = SpecInfo( @@ -2517,7 +2517,7 @@ def test_wf_shell_cmd_state_1(plugin, tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) wf.add( @@ -2585,7 +2585,7 @@ def test_wf_shell_cmd_ndst_1(plugin, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_input_spec2 = SpecInfo( @@ -2615,7 +2615,7 @@ def test_wf_shell_cmd_ndst_1(plugin, tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) wf.add( @@ -2884,7 +2884,7 @@ def test_shell_cmd_outputspec_5c(plugin, results_function, tmp_path): """ @attr.s(kw_only=True) - class MyOutputSpec(ShellOutputs): + class MyOutputDef(ShellOutputs): @staticmethod def gather_output(executable, output_dir): files = executable[1:] @@ -2895,7 +2895,7 @@ def gather_output(executable, output_dir): shelly = ShellTask( name="shelly", executable=["touch", "newfile_tmp1.txt", "newfile_tmp2.txt"], - output_spec=SpecInfo(name="Output", bases=(MyOutputSpec,)), + output_spec=SpecInfo(name="Output", bases=(MyOutputDef,)), cache_dir=tmp_path, ) @@ -3015,7 +3015,7 @@ def test_shell_cmd_outputspec_7(tmp_path, plugin, results_function): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_output_spec = SpecInfo( @@ -3091,7 +3091,7 @@ def test_shell_cmd_outputspec_7a(tmp_path, plugin, results_function): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_output_spec = SpecInfo( @@ -3288,7 +3288,7 @@ def get_lowest_directory(directory_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_output_spec = SpecInfo( @@ -3421,7 +3421,7 @@ def test_shell_cmd_inputspec_outputspec_1(): {"help_string": "2nd creadted file", "argstr": "", "position": 2}, ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_output_spec = SpecInfo( @@ -3475,7 +3475,7 @@ def test_shell_cmd_inputspec_outputspec_1a(): {"help_string": "2nd creadted file", "argstr": "", "position": 2}, ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_output_spec = SpecInfo( @@ -3528,7 +3528,7 @@ def test_shell_cmd_inputspec_outputspec_2(): {"help_string": "2nd creadted file", "argstr": "", "position": 2}, ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_output_spec = SpecInfo( @@ -3595,7 +3595,7 @@ def test_shell_cmd_inputspec_outputspec_2a(): {"help_string": "2nd creadted file", "argstr": "", "position": 2}, ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_output_spec = SpecInfo( @@ -3671,7 +3671,7 @@ def test_shell_cmd_inputspec_outputspec_3(): ), ("additional_inp", int, {"help_string": "additional inp"}), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_output_spec = SpecInfo( @@ -3732,7 +3732,7 @@ def test_shell_cmd_inputspec_outputspec_3a(): ), ("additional_inp", str, {"help_string": "additional inp"}), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_output_spec = SpecInfo( @@ -3801,7 +3801,7 @@ def test_shell_cmd_inputspec_outputspec_4(): ), ("additional_inp", int, {"help_string": "additional inp"}), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_output_spec = SpecInfo( @@ -3856,7 +3856,7 @@ def test_shell_cmd_inputspec_outputspec_4a(): ), ("additional_inp", int, {"help_string": "additional inp"}), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_output_spec = SpecInfo( @@ -3907,7 +3907,7 @@ def test_shell_cmd_inputspec_outputspec_5(): ("additional_inp_A", int, {"help_string": "additional inp A"}), ("additional_inp_B", str, {"help_string": "additional inp B"}), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_output_spec = SpecInfo( @@ -3961,7 +3961,7 @@ def test_shell_cmd_inputspec_outputspec_5a(): ("additional_inp_A", str, {"help_string": "additional inp A"}), ("additional_inp_B", int, {"help_string": "additional inp B"}), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_output_spec = SpecInfo( @@ -4015,7 +4015,7 @@ def test_shell_cmd_inputspec_outputspec_5b(): ("additional_inp_A", str, {"help_string": "additional inp A"}), ("additional_inp_B", str, {"help_string": "additional inp B"}), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_output_spec = SpecInfo( @@ -4067,7 +4067,7 @@ def test_shell_cmd_inputspec_outputspec_6_except(): ), ("additional_inp_A", str, {"help_string": "additional inp A"}), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_output_spec = SpecInfo( @@ -4328,7 +4328,7 @@ def change_name(file): # ("output_biascorrected", bool, # attr.ib(metadata={"help_string": 'output restored image (bias-corrected image)', "argstr": '-B'})), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # TODO: not sure why this has to be string @@ -4381,7 +4381,7 @@ def test_shell_cmd_optional_output_file1(tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_cp = ShellTask( @@ -4421,7 +4421,7 @@ def test_shell_cmd_optional_output_file2(tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_cp = ShellTask( @@ -4459,7 +4459,7 @@ def test_shell_cmd_non_existing_outputs_1(tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) out_spec = SpecInfo( name="Output", @@ -4521,7 +4521,7 @@ def test_shell_cmd_non_existing_outputs_2(tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) out_spec = SpecInfo( name="Output", @@ -4587,7 +4587,7 @@ def test_shell_cmd_non_existing_outputs_3(tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) out_spec = SpecInfo( name="Output", @@ -4654,7 +4654,7 @@ def test_shell_cmd_non_existing_outputs_4(tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) out_spec = SpecInfo( name="Output", @@ -4719,7 +4719,7 @@ def test_shell_cmd_non_existing_outputs_multi_1(tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) out_spec = SpecInfo( name="Output", @@ -4773,7 +4773,7 @@ def test_shell_cmd_non_existing_outputs_multi_2(tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) out_spec = SpecInfo( name="Output", @@ -4857,7 +4857,7 @@ def spec_info(formatter): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) def formatter_1(inputs): @@ -4970,7 +4970,7 @@ def spec_info(formatter): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # asking for specific inputs @@ -5023,7 +5023,7 @@ def test_shellcommand_error_msg(tmp_path): {"help_string": "a dummy string", "argstr": "", "mandatory": True}, ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( diff --git a/pydra/engine/tests/test_shelltask_inputspec.py b/pydra/engine/tests/test_shelltask_inputspec.py index b75c20a8a..e3b662af4 100644 --- a/pydra/engine/tests/test_shelltask_inputspec.py +++ b/pydra/engine/tests/test_shelltask_inputspec.py @@ -6,7 +6,7 @@ from ..task import ShellTask from pydra.engine.specs import ( ShellOutputs, - ShellSpec, + ShellDef, File, ) from pydra.design import shell @@ -38,7 +38,7 @@ def test_shell_cmd_inputs_1(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -54,7 +54,7 @@ def test_shell_cmd_inputs_1a(): fields=[ ("inpA", attr.ib(type=str, metadata={"help_string": "inpA", "argstr": ""})) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -77,7 +77,7 @@ def test_shell_cmd_inputs_1b(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -101,7 +101,7 @@ def test_shell_cmd_inputs_1_st(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) ShellTask( @@ -135,7 +135,7 @@ def test_shell_cmd_inputs_2(): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -153,7 +153,7 @@ def test_shell_cmd_inputs_2a(): ("inpA", attr.ib(type=str, metadata={"help_string": "inpA", "argstr": ""})), ("inpB", attr.ib(type=str, metadata={"help_string": "inpB", "argstr": ""})), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -187,7 +187,7 @@ def test_shell_cmd_inputs_2_err(): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -220,7 +220,7 @@ def test_shell_cmd_inputs_2_noerr(): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask(executable="executable", inpA="inp1", input_spec=my_input_spec) @@ -248,7 +248,7 @@ def test_shell_cmd_inputs_3(): ), ("inpC", attr.ib(type=str, metadata={"help_string": "inpC", "argstr": ""})), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -276,7 +276,7 @@ def test_shell_cmd_inputs_argstr_1(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask(executable="executable", inpA="inp1", input_spec=my_input_spec) @@ -297,7 +297,7 @@ def test_shell_cmd_inputs_argstr_2(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # separate command into exec + args @@ -321,7 +321,7 @@ def test_shell_cmd_inputs_list_1(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -344,7 +344,7 @@ def test_shell_cmd_inputs_list_2(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -366,7 +366,7 @@ def test_shell_cmd_inputs_list_3(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -394,7 +394,7 @@ def test_shell_cmd_inputs_list_sep_1(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -424,7 +424,7 @@ def test_shell_cmd_inputs_list_sep_2(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -454,7 +454,7 @@ def test_shell_cmd_inputs_list_sep_2a(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -484,7 +484,7 @@ def test_shell_cmd_inputs_list_sep_3(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -514,7 +514,7 @@ def test_shell_cmd_inputs_list_sep_3a(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -544,7 +544,7 @@ def test_shell_cmd_inputs_sep_4(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask(executable="executable", inpA=["aaa"], input_spec=my_input_spec) @@ -569,7 +569,7 @@ def test_shell_cmd_inputs_sep_4a(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask(executable="executable", inpA="aaa", input_spec=my_input_spec) @@ -593,7 +593,7 @@ def test_shell_cmd_inputs_format_1(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask(executable="executable", inpA="aaa", input_spec=my_input_spec) @@ -617,7 +617,7 @@ def test_shell_cmd_inputs_format_2(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -645,7 +645,7 @@ def test_shell_cmd_inputs_format_3(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask(executable="executable", inpA=0.007, input_spec=my_input_spec) @@ -670,7 +670,7 @@ def test_shell_cmd_inputs_mandatory_1(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask(executable="executable", input_spec=my_input_spec) @@ -714,7 +714,7 @@ def test_shell_cmd_inputs_not_given_1(): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask(name="shelly", executable="executable", input_spec=my_input_spec) @@ -753,7 +753,7 @@ def test_shell_cmd_inputs_template_1(): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask(executable="executable", input_spec=my_input_spec, inpA="inpA") @@ -792,7 +792,7 @@ def test_shell_cmd_inputs_template_1a(): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask(executable="executable", input_spec=my_input_spec, inpA="inpA") @@ -826,7 +826,7 @@ def test_shell_cmd_inputs_template_2(): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask(executable="executable", input_spec=my_input_spec) @@ -904,7 +904,7 @@ def test_shell_cmd_inputs_template_3(tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -984,7 +984,7 @@ def test_shell_cmd_inputs_template_3a(): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -1060,7 +1060,7 @@ def test_shell_cmd_inputs_template_4(): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask(executable="executable", input_spec=my_input_spec, inpA="inpA") @@ -1087,7 +1087,7 @@ def test_shell_cmd_inputs_template_5_ex(): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask(executable="executable", input_spec=my_input_spec, outAB="outAB") @@ -1130,7 +1130,7 @@ def test_shell_cmd_inputs_template_6(): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # no input for outA (and no default value), so the output is created whenever the @@ -1191,7 +1191,7 @@ def test_shell_cmd_inputs_template_6a(): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # no input for outA, but default is False, so the outA shouldn't be used @@ -1249,7 +1249,7 @@ def test_shell_cmd_inputs_template_7(tmp_path: Path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) inpA_file = tmp_path / "a_file.txt" @@ -1298,7 +1298,7 @@ def test_shell_cmd_inputs_template_7a(tmp_path: Path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) inpA_file = tmp_path / "a_file.txt" @@ -1347,7 +1347,7 @@ def test_shell_cmd_inputs_template_7b(tmp_path: Path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) inpA_file = tmp_path / "a_file.txt" @@ -1393,7 +1393,7 @@ def test_shell_cmd_inputs_template_8(tmp_path: Path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) inpA_file = tmp_path / "a_file.t" @@ -1453,7 +1453,7 @@ def test_shell_cmd_inputs_template_9(tmp_path: Path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) inpA_file = tmp_path / "inpA.t" @@ -1515,7 +1515,7 @@ def test_shell_cmd_inputs_template_9a(tmp_path: Path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) inpA_file = tmp_path / "inpA.t" @@ -1577,7 +1577,7 @@ def test_shell_cmd_inputs_template_9b_err(tmp_path: Path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) inpA_file = tmp_path / "inpA.t" @@ -1641,7 +1641,7 @@ def test_shell_cmd_inputs_template_9c_err(tmp_path: Path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) inpA_file = tmp_path / "inpA.t" @@ -1689,7 +1689,7 @@ def test_shell_cmd_inputs_template_10(): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask(executable="executable", input_spec=my_input_spec, inpA=3.3456) @@ -1701,7 +1701,7 @@ def test_shell_cmd_inputs_template_10(): def test_shell_cmd_inputs_template_requires_1(): - """Given an input specification with a templated output file subject to required fields, + """Given an input definition with a templated output file subject to required fields, ensure the field is set only when all requirements are met.""" my_input_spec = SpecInfo( @@ -1738,7 +1738,7 @@ def test_shell_cmd_inputs_template_requires_1(): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) # When requirements are not met. @@ -1787,7 +1787,7 @@ def template_fun(inputs): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask(executable="executable", input_spec=my_input_spec, inpA="inpA") @@ -1845,7 +1845,7 @@ def template_fun(inputs): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( @@ -1890,7 +1890,7 @@ def test_shell_cmd_inputs_template_1_st(): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) inpA = ["inpA_1", "inpA_2"] @@ -2085,7 +2085,7 @@ def test_shell_cmd_inputs_denoise_image( ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) my_input_file = tmp_path / "a_file.ext" @@ -2167,7 +2167,7 @@ def test_shell_cmd_inputs_denoise_image( @shell.define -class SimpleTaskXor(ShellSpec["SimpleTaskXor.Outputs"]): +class SimpleTaskXor(ShellDef["SimpleTaskXor.Outputs"]): input_1: str = shell.arg( help_string="help", diff --git a/pydra/engine/tests/test_singularity.py b/pydra/engine/tests/test_singularity.py index 7eec9b01d..247e36dfb 100644 --- a/pydra/engine/tests/test_singularity.py +++ b/pydra/engine/tests/test_singularity.py @@ -5,7 +5,7 @@ from ..task import ShellTask from ..submitter import Submitter -from ..specs import ShellOutputs, File, ShellSpec +from ..specs import ShellOutputs, File, ShellDef from ..environments import Singularity @@ -219,7 +219,7 @@ def test_singularity_inputspec_1(plugin, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) singu = ShellTask( @@ -260,7 +260,7 @@ def test_singularity_inputspec_1a(plugin, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) singu = ShellTask( @@ -317,7 +317,7 @@ def test_singularity_inputspec_2(plugin, tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) singu = ShellTask( @@ -377,7 +377,7 @@ def test_singularity_inputspec_2a_except(plugin, tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) singu = ShellTask( @@ -437,7 +437,7 @@ def test_singularity_inputspec_2a(plugin, tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) singu = ShellTask( @@ -494,7 +494,7 @@ def test_singularity_cmd_inputspec_copyfile_1(plugin, tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) singu = ShellTask( @@ -550,7 +550,7 @@ def test_singularity_inputspec_state_1(tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) singu = ShellTask( @@ -600,7 +600,7 @@ def test_singularity_inputspec_state_1b(plugin, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) singu = ShellTask( @@ -643,7 +643,7 @@ def test_singularity_wf_inputspec_1(plugin, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) wf = Workflow(name="wf", input_spec=["cmd", "file"], cache_dir=tmp_path) @@ -699,7 +699,7 @@ def test_singularity_wf_state_inputspec_1(plugin, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) wf = Workflow(name="wf", input_spec=["cmd", "file"], cache_dir=tmp_path) @@ -756,7 +756,7 @@ def test_singularity_wf_ndst_inputspec_1(plugin, tmp_path): ), ) ], - bases=(ShellSpec,), + bases=(ShellDef,), ) wf = Workflow(name="wf", input_spec=["cmd", "file"], cache_dir=tmp_path) diff --git a/pydra/engine/tests/test_specs.py b/pydra/engine/tests/test_specs.py index 6e1bf8f95..17ce176c8 100644 --- a/pydra/engine/tests/test_specs.py +++ b/pydra/engine/tests/test_specs.py @@ -10,7 +10,7 @@ File, Runtime, Result, - ShellSpec, + ShellDef, ) from pydra.engine.workflow.lazy import ( LazyInField, @@ -29,7 +29,7 @@ def test_basespec(): - spec = BaseSpec() + spec = BaseDef() assert spec.hash == "0b1d98df22ecd1733562711c205abca2" @@ -50,8 +50,8 @@ def test_result(): def test_shellspec(): with pytest.raises(TypeError): - spec = ShellSpec() - spec = ShellSpec(executable="ls") # (executable, args) + spec = ShellDef() + spec = ShellDef(executable="ls") # (executable, args) assert hasattr(spec, "executable") assert hasattr(spec, "args") @@ -63,7 +63,7 @@ class Input: inp_b: str = "B" def __init__(self): - class InpSpec: + class InpDef: def __init__(self): self.fields = [("inp_a", int), ("inp_b", int)] @@ -73,7 +73,7 @@ def __init__(self): self.name = "tn" self.inputs = self.Input() - self.input_spec = InpSpec() + self.input_spec = InpDef() self.output_spec = Outputs() self.output_names = ["out_a"] self.state = None @@ -138,14 +138,14 @@ def test_input_file_hash_1(tmp_path): os.chdir(tmp_path) outfile = "test.file" fields = [("in_file", ty.Any)] - input_spec = SpecInfo(name="Inputs", fields=fields, bases=(BaseSpec,)) + input_spec = SpecInfo(name="Inputs", fields=fields, bases=(BaseDef,)) inputs = make_klass(input_spec) assert inputs(in_file=outfile).hash == "9a106eb2830850834d9b5bf098d5fa85" with open(outfile, "w") as fp: fp.write("test") fields = [("in_file", File)] - input_spec = SpecInfo(name="Inputs", fields=fields, bases=(BaseSpec,)) + input_spec = SpecInfo(name="Inputs", fields=fields, bases=(BaseDef,)) inputs = make_klass(input_spec) assert inputs(in_file=outfile).hash == "02fa5f6f1bbde7f25349f54335e1adaf" @@ -156,7 +156,7 @@ def test_input_file_hash_2(tmp_path): with open(file, "w") as f: f.write("hello") - input_spec = SpecInfo(name="Inputs", fields=[("in_file", File)], bases=(BaseSpec,)) + input_spec = SpecInfo(name="Inputs", fields=[("in_file", File)], bases=(BaseDef,)) inputs = make_klass(input_spec) # checking specific hash value @@ -186,7 +186,7 @@ def test_input_file_hash_2a(tmp_path): f.write("hello") input_spec = SpecInfo( - name="Inputs", fields=[("in_file", ty.Union[File, int])], bases=(BaseSpec,) + name="Inputs", fields=[("in_file", ty.Union[File, int])], bases=(BaseDef,) ) inputs = make_klass(input_spec) @@ -221,7 +221,7 @@ def test_input_file_hash_3(tmp_path): f.write("hello") input_spec = SpecInfo( - name="Inputs", fields=[("in_file", File), ("in_int", int)], bases=(BaseSpec,) + name="Inputs", fields=[("in_file", File), ("in_int", int)], bases=(BaseDef,) ) inputs = make_klass(input_spec) @@ -279,7 +279,7 @@ def test_input_file_hash_4(tmp_path): input_spec = SpecInfo( name="Inputs", fields=[("in_file", ty.List[ty.List[ty.Union[int, File]]])], - bases=(BaseSpec,), + bases=(BaseDef,), ) inputs = make_klass(input_spec) @@ -316,7 +316,7 @@ def test_input_file_hash_5(tmp_path): input_spec = SpecInfo( name="Inputs", fields=[("in_file", ty.List[ty.Dict[ty.Any, ty.Union[File, int]]])], - bases=(BaseSpec,), + bases=(BaseDef,), ) inputs = make_klass(input_spec) diff --git a/pydra/engine/tests/test_task.py b/pydra/engine/tests/test_task.py index acea1497a..5a7e0d631 100644 --- a/pydra/engine/tests/test_task.py +++ b/pydra/engine/tests/test_task.py @@ -17,7 +17,7 @@ MultiOutputObj, ) from ..specs import ( - ShellSpec, + ShellDef, File, ) from pydra.utils.hash import hash_function @@ -592,7 +592,7 @@ def testfunc(a): my_input_spec = SpecInfo( name="Input", fields=[("a", attr.ib(type=float, metadata={"help_string": "input a"}))], - bases=(FunctionSpec,), + bases=(FunctionDef,), ) funky = testfunc(a=3.5, input_spec=my_input_spec) @@ -611,7 +611,7 @@ def testfunc(a): my_input_spec = SpecInfo( name="Input", fields=[("a", attr.ib(type=int, metadata={"help_string": "input a"}))], - bases=(FunctionSpec,), + bases=(FunctionDef,), ) with pytest.raises(TypeError): testfunc(a=3.5, input_spec=my_input_spec) @@ -634,7 +634,7 @@ def testfunc(a): attr.ib(type=float, metadata={"position": 1, "help_string": "input a"}), ) ], - bases=(FunctionSpec,), + bases=(FunctionDef,), ) with pytest.raises(AttributeError, match="only these keys are supported"): testfunc(a=3.5, input_spec=my_input_spec) @@ -649,7 +649,7 @@ def test_input_spec_func_1d_except(): def testfunc(a): return a - my_input_spec = SpecInfo(name="Input", fields=[], bases=(FunctionSpec,)) + my_input_spec = SpecInfo(name="Input", fields=[], bases=(FunctionDef,)) funky = testfunc(a=3.5, input_spec=my_input_spec) with pytest.raises(TypeError, match="missing 1 required positional argument"): funky() @@ -667,7 +667,7 @@ def testfunc(a: int): my_input_spec = SpecInfo( name="Input", fields=[("a", attr.ib(type=float, metadata={"help_string": "input a"}))], - bases=(FunctionSpec,), + bases=(FunctionDef,), ) funky = testfunc(a=3.5, input_spec=my_input_spec) @@ -687,7 +687,7 @@ def testfunc(a: int): my_input_spec = SpecInfo( name="Input", fields=[("a", float, {"help_string": "input a"})], - bases=(FunctionSpec,), + bases=(FunctionDef,), ) funky = testfunc(a=3.5, input_spec=my_input_spec) @@ -714,7 +714,7 @@ def testfunc(a): ), ) ], - bases=(FunctionSpec,), + bases=(FunctionDef,), ) funky = testfunc(a=2, input_spec=my_input_spec) @@ -741,7 +741,7 @@ def testfunc(a): ), ) ], - bases=(FunctionSpec,), + bases=(FunctionDef,), ) with pytest.raises(ValueError, match="value of a has to be"): @@ -773,7 +773,7 @@ def testfunc(a, b=1): ), ), ], - bases=(FunctionSpec,), + bases=(FunctionDef,), ) funky = testfunc(a=2, input_spec=my_input_spec) @@ -801,7 +801,7 @@ def testfunc(a, b=1): ), ("b", attr.ib(type=int, default=10, metadata={"help_string": "input b"})), ], - bases=(FunctionSpec,), + bases=(FunctionDef,), ) funky = testfunc(a=2, input_spec=my_input_spec) @@ -823,7 +823,7 @@ def testfunc(a): fields=[ ("a", attr.ib(type=MultiInputObj, metadata={"help_string": "input a"})) ], - bases=(FunctionSpec,), + bases=(FunctionDef,), ) funky = testfunc(a=3.5, input_spec=my_input_spec) @@ -842,7 +842,7 @@ def testfunc(a): my_output_spec = SpecInfo( name="Output", fields=[("out1", attr.ib(type=float, metadata={"help_string": "output"}))], - bases=(BaseSpec,), + bases=(BaseDef,), ) funky = testfunc(a=3.5, output_spec=my_output_spec) @@ -862,7 +862,7 @@ def testfunc(a): my_output_spec = SpecInfo( name="Output", fields=[("out1", attr.ib(type=int, metadata={"help_string": "output"}))], - bases=(BaseSpec,), + bases=(BaseDef,), ) funky = testfunc(a=3.5, output_spec=my_output_spec) @@ -882,7 +882,7 @@ def testfunc(a) -> int: my_output_spec = SpecInfo( name="Output", fields=[("out1", attr.ib(type=float, metadata={"help_string": "output"}))], - bases=(BaseSpec,), + bases=(BaseDef,), ) funky = testfunc(a=3.5, output_spec=my_output_spec) @@ -903,7 +903,7 @@ def testfunc(a) -> int: my_output_spec = SpecInfo( name="Output", fields=[("out1", float, {"help_string": "output"})], - bases=(BaseSpec,), + bases=(BaseDef,), ) funky = testfunc(a=3.5, output_spec=my_output_spec) @@ -928,7 +928,7 @@ def testfunc(a, b): attr.ib(type=MultiOutputObj, metadata={"help_string": "output"}), ) ], - bases=(BaseSpec,), + bases=(BaseDef,), ) funky = testfunc(a=3.5, b=1, output_spec=my_output_spec) @@ -953,7 +953,7 @@ def testfunc(a): attr.ib(type=MultiOutputObj, metadata={"help_string": "output"}), ) ], - bases=(BaseSpec,), + bases=(BaseDef,), ) funky = testfunc(a=3.5, output_spec=my_output_spec) @@ -1135,7 +1135,7 @@ def test_audit_shellcommandtask_file(tmp_path): ), ), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) shelly = ShellTask( name="shelly", diff --git a/pydra/engine/tests/test_workflow.py b/pydra/engine/tests/test_workflow.py index cfbaa4ad6..9384e1de6 100644 --- a/pydra/engine/tests/test_workflow.py +++ b/pydra/engine/tests/test_workflow.py @@ -35,7 +35,7 @@ ) from ..submitter import Submitter from pydra.design import python -from ..specs import ShellSpec +from ..specs import ShellDef from pydra.utils import exc_info_matches @@ -51,7 +51,7 @@ def test_wf_specinfo_input_spec(): ("a", str, "", {"mandatory": True}), ("b", dict, {"foo": 1, "bar": False}, {"mandatory": False}), ], - bases=(BaseSpec,), + bases=(BaseDef,), ) wf = Workflow( name="workflow", @@ -66,10 +66,10 @@ def test_wf_specinfo_input_spec(): fields=[ ("a", str, {"mandatory": True}), ], - bases=(ShellSpec,), + bases=(ShellDef,), ) with pytest.raises( - ValueError, match="Provided SpecInfo must have BaseSpec as its base." + ValueError, match="Provided SpecInfo must have BaseDef as its base." ): Workflow(name="workflow", input_spec=bad_input_spec) @@ -243,7 +243,7 @@ def test_wf_1_call_exception(plugin, tmpdir): with Submitter(plugin=plugin) as sub: with pytest.raises(Exception) as e: wf(submitter=sub, plugin=plugin) - assert "Specify submitter OR plugin" in str(e.value) + assert "Defify submitter OR plugin" in str(e.value) def test_wf_1_inp_in_call(tmpdir): diff --git a/pydra/engine/workers.py b/pydra/engine/workers.py index e6bf3cced..8dd09e084 100644 --- a/pydra/engine/workers.py +++ b/pydra/engine/workers.py @@ -941,10 +941,10 @@ def make_spec(self, cmd=None, arg=None): Returns ------- - psij.JobSpec + psij.JobDef PSI/J job specification. """ - spec = self.psij.JobSpec() + spec = self.psij.JobDef() spec.executable = cmd spec.arguments = arg @@ -956,7 +956,7 @@ def make_job(self, spec, attributes): Parameters ---------- - spec : psij.JobSpec + spec : psij.JobDef PSI/J job specification. attributes : any Job attributes. diff --git a/pydra/engine/workflow/base.py b/pydra/engine/workflow/base.py index cf649ebd4..ebe683531 100644 --- a/pydra/engine/workflow/base.py +++ b/pydra/engine/workflow/base.py @@ -4,7 +4,7 @@ from typing_extensions import Self import attrs from pydra.engine.helpers import list_fields, attrs_values, is_lazy -from pydra.engine.specs import TaskSpec, TaskOutputs, WorkflowOutputs +from pydra.engine.specs import TaskDef, TaskOutputs, WorkflowOutputs from .lazy import LazyInField from pydra.utils.hash import hash_function from pydra.utils.typing import TypeParser, StateArray @@ -17,29 +17,29 @@ @attrs.define(auto_attribs=False) class Workflow(ty.Generic[WorkflowOutputsType]): - """A workflow, constructed from a workflow specification + """A workflow, constructed from a workflow definition Parameters ---------- name : str The name of the workflow - inputs : TaskSpec - The input specification of the workflow - outputs : TaskSpec - The output specification of the workflow + inputs : TaskDef + The input definition of the workflow + outputs : TaskDef + The output definition of the workflow """ name: str = attrs.field() - inputs: TaskSpec[WorkflowOutputsType] = attrs.field() + inputs: TaskDef[WorkflowOutputsType] = attrs.field() outputs: WorkflowOutputsType = attrs.field() _nodes: dict[str, Node] = attrs.field(factory=dict) @classmethod def construct( cls, - spec: TaskSpec[WorkflowOutputsType], + spec: TaskDef[WorkflowOutputsType], ) -> Self: - """Construct a workflow from a specification, caching the constructed worklow""" + """Construct a workflow from a definition, caching the constructed worklow""" lazy_inputs = [f for f in list_fields(type(spec)) if f.lazy] @@ -129,21 +129,21 @@ def clear_cache(cls): """Clear the cache of constructed workflows""" cls._constructed.clear() - def add(self, task_spec: TaskSpec[OutputsType], name=None) -> OutputsType: + def add(self, task_spec: TaskDef[OutputsType], name=None) -> OutputsType: """Add a node to the workflow Parameters ---------- - task_spec : TaskSpec - The specification of the task to add to the workflow as a node + task_spec : TaskDef + The definition of the task to add to the workflow as a node name : str, optional - The name of the node, by default it will be the name of the task specification + The name of the node, by default it will be the name of the task definition class Returns ------- OutputType - The outputs specification of the node + The outputs definition of the node """ if name is None: name = type(task_spec).__name__ diff --git a/pydra/engine/workflow/node.py b/pydra/engine/workflow/node.py index 2a3daef98..2920e5a07 100644 --- a/pydra/engine/workflow/node.py +++ b/pydra/engine/workflow/node.py @@ -5,7 +5,7 @@ import attrs from pydra.utils.typing import TypeParser, StateArray from . import lazy -from ..specs import TaskSpec, TaskOutputs, WorkflowSpec +from ..specs import TaskDef, TaskOutputs, WorkflowDef from ..task import Task from ..helpers import ensure_list, attrs_values, is_lazy, load_result, create_checksum from pydra.utils.hash import hash_function @@ -32,12 +32,12 @@ class Node(ty.Generic[OutputType]): ---------- name : str The name of the node - inputs : TaskSpec - The specification of the node + inputs : TaskDef + The definition of the node """ name: str - _spec: TaskSpec[OutputType] + _spec: TaskDef[OutputType] _workflow: "Workflow" = attrs.field(default=None, eq=False, hash=False) _lzout: OutputType | None = attrs.field( init=False, default=None, eq=False, hash=False @@ -148,7 +148,7 @@ def split( Returns ------- - self : TaskSpec + self : TaskDef a reference to the task """ self._check_if_outputs_have_been_used("the node cannot be split or combined") @@ -222,7 +222,7 @@ def combine( Returns ------- - self : TaskSpec + self : TaskDef a reference to the task """ if not isinstance(combiner, (str, list)): @@ -345,7 +345,7 @@ def _checksum_states(self, state_index=None): # that might be important for outer splitter of input variable with big files # the file can be changed with every single index even if there are only two files input_hash = inputs_copy.hash - if isinstance(self._spec, WorkflowSpec): + if isinstance(self._spec, WorkflowDef): con_hash = hash_function(self._connections) # TODO: hash list is not used hash_list = [input_hash, con_hash] # noqa: F841 diff --git a/pydra/utils/tests/utils.py b/pydra/utils/tests/utils.py index 2cd5ad357..8bf993292 100644 --- a/pydra/utils/tests/utils.py +++ b/pydra/utils/tests/utils.py @@ -31,7 +31,7 @@ def generic_func_task(in_file: File) -> File: @shell.define -class GenericShellTask(specs.ShellSpec["GenericShellTask.Outputs"]): +class GenericShellTask(specs.ShellDef["GenericShellTask.Outputs"]): """class with customized input and executables""" in_file: File = shell.arg( @@ -57,7 +57,7 @@ def specific_func_task(in_file: MyFormatX) -> MyFormatX: @shell.define -class SpecificShellTask(specs.ShellSpec["SpecificShellTask.Outputs"]): +class SpecificShellTask(specs.ShellDef["SpecificShellTask.Outputs"]): executable = "echo" in_file: MyFormatX = shell.arg( diff --git a/pydra/utils/typing.py b/pydra/utils/typing.py index 2ce2efd1f..976f59b43 100644 --- a/pydra/utils/typing.py +++ b/pydra/utils/typing.py @@ -409,7 +409,7 @@ def coerce_obj(obj, type_): try: return expand_and_coerce(object_, self.pattern) except TypeError as e: - # Special handling for MultiInputObjects (which are annoying) + # Defial handling for MultiInputObjects (which are annoying) if isinstance(self.pattern, tuple) and self.pattern[0] == MultiInputObj: # Attempt to coerce the object into arg type of the MultiInputObj first, # and if that fails, try to coerce it into a list of the arg type @@ -588,7 +588,7 @@ def check_sequence(tp_args, pattern_args): try: return expand_and_check(type_, self.pattern) except TypeError as e: - # Special handling for MultiInputObjects (which are annoying) + # Defial handling for MultiInputObjects (which are annoying) if not isinstance(self.pattern, tuple) or self.pattern[0] != MultiInputObj: raise e # Attempt to coerce the object into arg type of the MultiInputObj first,