feat(scripts): added composite tasks support (#1117)

docs/docs/usage/scripts.md

@@ -35,7 +35,7 @@ $ pdm run start -h 0.0.0.0
 Flask server started at http://0.0.0.0:54321
 ```
 
-PDM supports 3 types of scripts:
+PDM supports 4 types of scripts:
 
 ### `cmd`
 
@@ -85,6 +85,32 @@ The function can be supplied with literal arguments:
 foobar = {call = "foo_package.bar_module:main('dev')"}
 ```
 
+### `composite`
+
+This script kind execute other defined scripts:
+
+```toml
+[tool.pdm.scripts]
+lint = "flake8"
+test = "pytest"
+all = {composite = ["lint", "test"]}
+```
+
+Running `pdm run all` will run `lint` first and then `test` if `lint` succeeded.
+
+You can also provide arguments to the called scripts:
+
+```toml
+[tool.pdm.scripts]
+lint = "flake8"
+test = "pytest"
+all = {composite = ["lint mypackage/", "test -v tests/"]}
+```
+
+!!! note
+    Argument passed on the command line are given to each called task.
+
+
 ### `env`
 
 All environment variables set in the current shell can be seen by `pdm run` and will be expanded when executed.
@@ -98,6 +124,9 @@ start.env = {FOO = "bar", FLASK_ENV = "development"}
 
 Note how we use [TOML's syntax](https://github.com/toml-lang/toml) to define a composite dictionary.
 
+!!! note
+    Environment variables specified on a composite task level will override those defined by called tasks.
+
 ### `env_file`
 
 You can also store all environment variables in a dotenv file and let PDM read it:
@@ -108,6 +137,9 @@ start.cmd = "flask run -p 54321"
 start.env_file = ".env"
 ```
 
+!!! note
+    A dotenv file specified on a composite task level will override those defined by called tasks.
+
 ### `site_packages`
 
 To make sure the running environment is properly isolated from the outer Python interpreter,
@@ -183,3 +215,7 @@ Under certain situations PDM will look for some special hook scripts for executi
     If there exists an `install` scripts under `[tool.pdm.scripts]` table, `pre_install`
     scripts can be triggered by both `pdm install` and `pdm run install`. So it is
     recommended to not use the preserved names.
+
+!!! note
+    Composite tasks can also have pre and post scripts.
+    Called tasks will run their own pre and post scripts.

news/1117.feature.md

+Add a `composite` script kind allowing to run multiple defined scripts in a single command as well as reusing scripts but overriding `env` or `env_file`.

pdm/cli/commands/run.py

@@ -26,6 +26,19 @@ class TaskOptions(TypedDict, total=False):
     site_packages: bool
 
 
+def exec_opts(*options: TaskOptions | None) -> dict[str, Any]:
+    return dict(
+        env={k: v for opts in options if opts for k, v in opts.get("env", {}).items()},
+        **{
+            k: v
+            for opts in options
+            if opts
+            for k, v in opts.items()
+            if k not in ("env", "help")
+        },
+    )
+
+
 class Task(NamedTuple):
     kind: str
     name: str
@@ -39,7 +52,7 @@ class Task(NamedTuple):
 class TaskRunner:
     """The task runner for pdm project"""
 
-    TYPES = ["cmd", "shell", "call"]
+    TYPES = ["cmd", "shell", "call", "composite"]
     OPTIONS = ["env", "env_file", "help", "site_packages"]
 
     def __init__(self, project: Project) -> None:
@@ -161,9 +174,10 @@ class TaskRunner:
         signal.signal(signal.SIGINT, s)
         return process.returncode
 
-    def _run_task(self, task: Task, args: Sequence[str] = ()) -> int:
+    def _run_task(
+        self, task: Task, args: Sequence[str] = (), opts: TaskOptions | None = None
+    ) -> int:
         kind, _, value, options = task
-        options.pop("help", None)
         shell = False
         if kind == "cmd":
             if not isinstance(value, list):
@@ -189,38 +203,51 @@ class TaskRunner:
                 f"import sys, {module} as {short_name};"
                 f"sys.exit({short_name}.{func})",
             ] + list(args)
-        if "env" in self.global_options:
-            options["env"] = {**self.global_options["env"], **options.get("env", {})}
-        options["env_file"] = options.get(
-            "env_file", self.global_options.get("env_file")
-        )
+        elif kind == "composite":
+            assert isinstance(value, list)
+
         self.project.core.ui.echo(
             f"Running {task}: [green]{str(args)}[/]",
             err=True,
             verbosity=termui.Verbosity.DETAIL,
         )
+        if kind == "composite":
+            for script in value:
+                splitted = shlex.split(script)
+                cmd = splitted[0]
+                subargs = splitted[1:] + args  # type: ignore
+                code = self.run(cmd, subargs, options)
+                if code != 0:
+                    return code
+            return code
         return self._run_process(
-            args, chdir=True, shell=shell, **options  # type: ignore
+            args,
+            chdir=True,
+            shell=shell,
+            **exec_opts(self.global_options, options, opts),
         )
 
-    def run(self, command: str, args: Sequence[str]) -> int:
+    def run(
+        self, command: str, args: Sequence[str], opts: TaskOptions | None = None
+    ) -> int:
         task = self._get_task(command)
         if task is not None:
             pre_task = self._get_task(f"pre_{command}")
             if pre_task is not None:
-                code = self._run_task(pre_task)
+                code = self._run_task(pre_task, opts=opts)
                 if code != 0:
                     return code
-            code = self._run_task(task, args)
+            code = self._run_task(task, args, opts=opts)
             if code != 0:
                 return code
             post_task = self._get_task(f"post_{command}")
             if post_task is not None:
-                code = self._run_task(post_task)
+                code = self._run_task(post_task, opts=opts)
             return code
         else:
             return self._run_process(
-                [command] + args, **self.global_options  # type: ignore
+                [command] + args,  # type: ignore
+                **exec_opts(self.global_options, opts),
             )
 
     def show_list(self) -> None:

tests/cli/test_run.py

@@ -5,10 +5,42 @@ import textwrap
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
+import pytest
+
 from pdm.cli.actions import PEP582_PATH
 from pdm.utils import cd
 
 
+@pytest.fixture
+def _vars(project):
+    (project.root / "vars.py").write_text(
+        textwrap.dedent(
+            """
+            import os
+            import sys
+            name = sys.argv[1]
+            vars = " ".join([f"{v}={os.getenv(v)}" for v in sys.argv[2:]])
+            print(f"{name} CALLED with {vars}" if vars else f"{name} CALLED")
+            """
+        )
+    )
+
+
+@pytest.fixture
+def _args(project):
+    (project.root / "args.py").write_text(
+        textwrap.dedent(
+            """
+            import os
+            import sys
+            name = sys.argv[1]
+            args = ", ".join(sys.argv[2:])
+            print(f"{name} CALLED with {args}" if args else f"{name} CALLED")
+            """
+        )
+    )
+
+
 def test_pep582_launcher_for_python_interpreter(project, local_finder, invoke):
     project.root.joinpath("main.py").write_text(
         "import first;print(first.first([0, False, 1, 2]))\n"
@@ -350,8 +382,213 @@ def test_pre_and_post_scripts(project, invoke, capfd):
         "post_test": "python -c \"print('POST test CALLED')\"",
     }
     project.write_pyproject()
+    capfd.readouterr()
     invoke(["run", "test"], strict=True, obj=project)
     out, _ = capfd.readouterr()
     assert "PRE test CALLED" in out
     assert "IN test CALLED" in out
     assert "POST test CALLED" in out
+
+
+def test_run_composite(project, invoke, capfd):
+    project.tool_settings["scripts"] = {
+        "first": "echo 'First CALLED'",
+        "second": {"shell": "echo 'Second CALLED'"},
+        "test": {"composite": ["first", "second"]},
+    }
+    project.write_pyproject()
+    capfd.readouterr()
+    invoke(["run", "test"], strict=True, obj=project)
+    out, _ = capfd.readouterr()
+    assert "First CALLED" in out
+    assert "Second CALLED" in out
+
+
+def test_composite_stops_on_first_failure(project, invoke, capfd):
+    project.tool_settings["scripts"] = {
+        "first": "echo 'First CALLED'",
+        "fail": "false",
+        "second": "echo 'Second CALLED'",
+        "test": {"composite": ["first", "fail", "second"]},
+    }
+    project.write_pyproject()
+    capfd.readouterr()
+    result = invoke(["run", "test"], obj=project)
+    assert result.exit_code == 1
+    out, _ = capfd.readouterr()
+    assert "First CALLED" in out
+    assert "Second CALLED" not in out
+
+
+def test_composite_inherit_env(project, invoke, capfd, _vars):
+    project.tool_settings["scripts"] = {
+        "first": {
+            "cmd": "python vars.py First VAR",
+            "env": {"VAR": "42"},
+        },
+        "second": {
+            "cmd": "python vars.py Second VAR",
+            "env": {"VAR": "42"},
+        },
+        "test": {"composite": ["first", "second"], "env": {"VAR": "overriden"}},
+    }
+    project.write_pyproject()
+    capfd.readouterr()
+    invoke(["run", "test"], strict=True, obj=project)
+    out, _ = capfd.readouterr()
+    assert "First CALLED with VAR=overriden" in out
+    assert "Second CALLED with VAR=overriden" in out
+
+
+def test_composite_fail_on_first_missing_task(project, invoke, capfd):
+    project.tool_settings["scripts"] = {
+        "first": "echo 'First CALLED'",
+        "second": "echo 'Second CALLED'",
+        "test": {"composite": ["first", "fail", "second"]},
+    }
+    project.write_pyproject()
+    capfd.readouterr()
+    result = invoke(["run", "test"], obj=project)
+    assert result.exit_code == 1
+    out, _ = capfd.readouterr()
+    assert "First CALLED" in out
+    assert "Second CALLED" not in out
+
+
+def test_composite_runs_all_hooks(project, invoke, capfd):
+    project.tool_settings["scripts"] = {
+        "test": {"composite": ["first", "second"]},
+        "pre_test": "echo 'Pre-Test CALLED'",
+        "post_test": "echo 'Post-Test CALLED'",
+        "first": "echo 'First CALLED'",
+        "pre_first": "echo 'Pre-First CALLED'",
+        "second": "echo 'Second CALLED'",
+        "post_second": "echo 'Post-Second CALLED'",
+    }
+    project.write_pyproject()
+    capfd.readouterr()
+    invoke(["run", "test"], strict=True, obj=project)
+    out, _ = capfd.readouterr()
+    assert "Pre-Test CALLED" in out
+    assert "Pre-First CALLED" in out
+    assert "First CALLED" in out
+    assert "Second CALLED" in out
+    assert "Post-Second CALLED" in out
+    assert "Post-Test CALLED" in out
+
+
+def test_composite_pass_parameters_to_subtasks(project, invoke, capfd, _args):
+    project.tool_settings["scripts"] = {
+        "test": {"composite": ["first", "second"]},
+        "pre_test": "python args.py Pre-Test",
+        "post_test": "python args.py Post-Test",
+        "first": "python args.py First",
+        "pre_first": "python args.py Pre-First",
+        "second": "python args.py Second",
+        "post_second": "python args.py Post-Second",
+    }
+    project.write_pyproject()
+    capfd.readouterr()
+    invoke(["run", "test", "param=value"], strict=True, obj=project)
+    out, _ = capfd.readouterr()
+    assert "Pre-Test CALLED" in out
+    assert "Pre-First CALLED" in out
+    assert "First CALLED with param=value" in out
+    assert "Second CALLED with param=value" in out
+    assert "Post-Second CALLED" in out
+    assert "Post-Test CALLED" in out
+
+
+def test_composite_can_pass_parameters(project, invoke, capfd, _args):
+    project.tool_settings["scripts"] = {
+        "test": {"composite": ["first param=first", "second param=second"]},
+        "pre_test": "python args.py Pre-Test",
+        "post_test": "python args.py Post-Test",
+        "first": "python args.py First",
+        "pre_first": "python args.py Pre-First",
+        "second": "python args.py Second",
+        "post_second": "python args.py Post-Second",
+    }
+    project.write_pyproject()
+    capfd.readouterr()
+    invoke(["run", "test"], strict=True, obj=project)
+    out, _ = capfd.readouterr()
+    assert "Pre-Test CALLED" in out
+    assert "Pre-First CALLED" in out
+    assert "First CALLED with param=first" in out
+    assert "Second CALLED with param=second" in out
+    assert "Post-Second CALLED" in out
+    assert "Post-Test CALLED" in out
+
+
+def test_composite_hooks_inherit_env(project, invoke, capfd, _vars):
+    project.tool_settings["scripts"] = {
+        "pre_task": {"cmd": "python vars.py Pre-Task VAR", "env": {"VAR": "42"}},
+        "task": "echo 'Task CALLED'",
+        "post_task": {"cmd": "python vars.py Post-Task VAR", "env": {"VAR": "42"}},
+        "test": {"composite": ["task"], "env": {"VAR": "overriden"}},
+    }
+    project.write_pyproject()
+    capfd.readouterr()
+    invoke(["run", "test"], strict=True, obj=project)
+    out, _ = capfd.readouterr()
+    assert "Pre-Task CALLED with VAR=overriden" in out
+    assert "Task CALLED" in out
+    assert "Post-Task CALLED with VAR=overriden" in out
+
+
+def test_composite_inherit_env_in_cascade(project, invoke, capfd, _vars):
+    project.tool_settings["scripts"] = {
+        "_": {"env": {"FOO": "BAR", "TIK": "TOK"}},
+        "pre_task": {
+            "cmd": "python vars.py Pre-Task VAR FOO TIK",
+            "env": {"VAR": "42", "FOO": "foobar"},
+        },
+        "task": {
+            "cmd": "python vars.py Task VAR FOO TIK",
+            "env": {"VAR": "42", "FOO": "foobar"},
+        },
+        "post_task": {
+            "cmd": "python vars.py Post-Task VAR FOO TIK",
+            "env": {"VAR": "42", "FOO": "foobar"},
+        },
+        "test": {"composite": ["task"], "env": {"VAR": "overriden"}},
+    }
+    project.write_pyproject()
+    capfd.readouterr()
+    invoke(["run", "test"], strict=True, obj=project)
+    out, _ = capfd.readouterr()
+    assert "Pre-Task CALLED with VAR=overriden FOO=foobar TIK=TOK" in out
+    assert "Task CALLED with VAR=overriden FOO=foobar TIK=TOK" in out
+    assert "Post-Task CALLED with VAR=overriden FOO=foobar TIK=TOK" in out
+
+
+def test_composite_inherit_dotfile(project, invoke, capfd, _vars):
+    (project.root / ".env").write_text("VAR=42")
+    (project.root / "override.env").write_text("VAR=overriden")
+    project.tool_settings["scripts"] = {
+        "pre_task": {"cmd": "python vars.py Pre-Task VAR", "env_file": ".env"},
+        "task": {"cmd": "python vars.py Task VAR", "env_file": ".env"},
+        "post_task": {"cmd": "python vars.py Post-Task VAR", "env_file": ".env"},
+        "test": {"composite": ["task"], "env_file": "override.env"},
+    }
+    project.write_pyproject()
+    capfd.readouterr()
+    invoke(["run", "test"], strict=True, obj=project)
+    out, _ = capfd.readouterr()
+    assert "Pre-Task CALLED with VAR=overriden" in out
+    assert "Task CALLED with VAR=overriden" in out
+    assert "Post-Task CALLED with VAR=overriden" in out
+
+
+def test_composite_can_have_commands(project, invoke, capfd):
+    project.tool_settings["scripts"] = {
+        "task": "echo 'Task CALLED'",
+        "test": {"composite": ["task", "echo 'Command CALLED'"]},
+    }
+    project.write_pyproject()
+    capfd.readouterr()
+    invoke(["run", "test"], strict=True, obj=project)
+    out, _ = capfd.readouterr()
+    assert "Task CALLED" in out
+    assert "Command CALLED" in out

tests/conftest.py

@@ -381,6 +381,7 @@ def invoke(core):
     runner = CliRunner(mix_stderr=False)
 
     def caller(args, strict=False, **kwargs):
+        __tracebackhide__ = True
         result = runner.invoke(
             core, args, catch_exceptions=not strict, prog_name="pdm", **kwargs
         )