From cc2ad1e2a156f877686f8ed3e2052815a9400037 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sat, 2 Jul 2022 18:23:37 +0800 Subject: [PATCH] feat: pdm venv integration (#1193) --- .github/workflows/ci.yml | 4 +- .gitignore | 2 +- README.md | 37 ++--- README_zh.md | 40 ++--- docs/docs/index.md | 151 +----------------- docs/docs/plugin/write.md | 2 +- docs/docs/usage/configuration.md | 49 +++--- docs/docs/usage/pep582.md | 162 ++++++++++++++++++++ docs/docs/usage/project.md | 39 ++--- docs/docs/usage/venv.md | 106 +++++++++++++ docs/mkdocs.yml | 6 +- news/1162.feature.md | 1 + news/1191.removal.md | 1 + pdm.lock | 10 +- pdm/cli/actions.py | 8 +- pdm/cli/commands/init.py | 38 ++++- pdm/cli/commands/plugin.py | 4 +- pdm/cli/commands/venv/__init__.py | 30 ++++ pdm/cli/commands/venv/activate.py | 62 ++++++++ pdm/cli/commands/venv/backends.py | 158 +++++++++++++++++++ pdm/cli/commands/venv/create.py | 57 +++++++ pdm/cli/commands/venv/list.py | 23 +++ pdm/cli/commands/venv/purge.py | 65 ++++++++ pdm/cli/commands/venv/remove.py | 47 ++++++ pdm/cli/commands/venv/utils.py | 80 ++++++++++ pdm/cli/completions/pdm.bash | 14 +- pdm/cli/completions/pdm.fish | 67 ++++---- pdm/cli/completions/pdm.ps1 | 2 +- pdm/models/environment.py | 4 +- pdm/project/config.py | 18 ++- pdm/project/core.py | 81 ++++++++-- pdm/termui.py | 1 + pdm/utils.py | 25 --- pyproject.toml | 25 +-- tests/cli/test_build.py | 5 +- tests/cli/test_config.py | 12 +- tests/cli/test_hooks.py | 6 +- tests/cli/test_init.py | 111 ++++++++++++++ tests/cli/test_others.py | 83 ---------- tests/cli/test_run.py | 4 +- tests/cli/test_venv.py | 246 ++++++++++++++++++++++++++++++ tests/conftest.py | 45 +++--- tests/test_project.py | 67 +++++++- tests/test_utils.py | 20 +-- 44 files changed, 1543 insertions(+), 475 deletions(-) create mode 100644 docs/docs/usage/pep582.md create mode 100644 docs/docs/usage/venv.md create mode 100644 news/1162.feature.md create mode 100644 news/1191.removal.md create mode 100644 pdm/cli/commands/venv/__init__.py create mode 100644 pdm/cli/commands/venv/activate.py create mode 100644 pdm/cli/commands/venv/backends.py create mode 100644 pdm/cli/commands/venv/create.py create mode 100644 pdm/cli/commands/venv/list.py create mode 100644 pdm/cli/commands/venv/purge.py create mode 100644 pdm/cli/commands/venv/remove.py create mode 100644 pdm/cli/commands/venv/utils.py create mode 100644 tests/cli/test_init.py create mode 100644 tests/cli/test_venv.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 724a3b98..1d3e4a25 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,11 +38,11 @@ jobs: install-via: [pip] arch: [x64] include: - - python-version: 3.9 + - python-version: "3.10" os: ubuntu-latest install-via: script arch: x64 - - python-version: 3.9 + - python-version: "3.10" os: windows-latest install-via: pip arch: x86 diff --git a/.gitignore b/.gitignore index f6367b1d..83b688b5 100644 --- a/.gitignore +++ b/.gitignore @@ -105,7 +105,7 @@ celerybeat.pid .env .venv env/ -venv/ +/venv/ ENV/ env.bak/ venv.bak/ diff --git a/README.md b/README.md index 1c70a392..2ccf9e01 100644 --- a/README.md +++ b/README.md @@ -30,23 +30,9 @@ with `Pipenv` or `Poetry` and don't want to introduce another package manager, just stick to it. But if you are missing something that is not present in those tools, you can probably find some goodness in `pdm`. -[PEP 582] proposes a project structure as below: - -``` -foo - __pypackages__ - 3.8 - lib - bottle - myscript.py -``` - -There is a `__pypackages__` directory in the project root to hold all dependent libraries, just like what `npm` does. -Read more about the specification [here](https://www.python.org/dev/peps/pep-0582/#specification). - ## Highlights of features -- [PEP 582] local package installer and runner, no virtualenv involved at all. +- Opt-in [PEP 582] support, no virtualenv involved at all. - Simple and fast dependency resolver, mainly for large binary distributions. - A [PEP 517] build backend. - [PEP 621] project metadata. @@ -59,7 +45,7 @@ Read more about the specification [here](https://www.python.org/dev/peps/pep-058 [pep 621]: https://www.python.org/dev/peps/pep-0621 [pnpm]: https://pnpm.io/motivation#saving-disk-space-and-boosting-installation-speed -## Why not virtualenv? +## What is PEP 582? The majority of Python packaging tools also act as virtualenv managers to gain the ability to isolate project environments. But things get tricky when it comes to nested venvs: One @@ -71,6 +57,20 @@ all those venvs and upgrade them if required. environments. It is a relatively new proposal and there are not many tools supporting it (one that does is [pyflow], but it is written with Rust and thus can't get much help from the big Python community and for the same reason it can't act as a [PEP 517] backend). +[PEP 582] proposes a project structure as below: + +``` +foo + __pypackages__ + 3.8 + lib + bottle + myscript.py +``` + +There is a `__pypackages__` directory in the project root to hold all dependent libraries, just like what `npm` does. +Read more about the specification [here](https://www.python.org/dev/peps/pep-0582/#specification). + ## Installation PDM requires python version 3.7 or higher. @@ -160,7 +160,7 @@ pdm init Answer the questions following the guide, and a PDM project with a `pyproject.toml` file will be ready to use. -**Install dependencies into the `__pypackages__` directory** +**Install dependencies** ```bash pdm add requests flask @@ -170,6 +170,9 @@ You can add multiple dependencies in the same command. After a while, check the **Run your script with [PEP 582] support** +By default, PDM will create `.venv` in the project root, when doing `pdm install` on an existing project, as other package managers do. +But you can make PEP 582 the default by `pdm config python.use_venv false`. To enable the full power of PEP 582, do the following steps to make the Python interpreter use it. + Suppose you have a script `app.py` placed next to the `__pypackages__` directory with the following content(taken from Flask's website): ```python diff --git a/README_zh.md b/README_zh.md index ff5e22a8..aaa67866 100644 --- a/README_zh.md +++ b/README_zh.md @@ -26,23 +26,9 @@ PDM 旨在成为下一代 Python 软件包管理工具。它最初是为个人 `poetry` 用着非常好，并不想引入一个新的包管理器，那么继续使用它们吧；但如果你发现有些东西这些 工具不支持，那么你很可能可以在 `pdm` 中找到。 -[PEP 582] 提出下面这种项目的目录结构： - -``` -foo - __pypackages__ - 3.8 - lib - bottle - myscript.py -``` - -项目目录中包含一个`__pypackages__`目录，用来放置所有依赖的库文件，就像`npm`的`node_modules`一样。 -你可以在[这里](https://www.python.org/dev/peps/pep-0582/#specification)阅读更多提案的细节。 - ## 主要特性 -- [PEP 582] 本地项目库目录，支持安装与运行命令，完全不需要虚拟环境。 +- [PEP 582] 支持，完全不需要虚拟环境。 - 一个简单且相对快速的依赖解析器，特别是对于大的二进制包发布。 - 兼容 [PEP 517] 的构建后端，用于构建发布包(源码格式与 wheel 格式) - 灵活且强大的插件系统 @@ -66,6 +52,20 @@ foo 没有很多相关的工具实现它，这其中就有 [pyflow]。但 pyflow 又是用 Rust 写的，不是所有 Python 的社区 都会用 Rust，这样就没法贡献代码，而且，基于同样的原因，pyflow 并不支持 [PEP 517] 构建。 +[PEP 582] 提出下面这种项目的目录结构： + +``` +foo + __pypackages__ + 3.8 + lib + bottle + myscript.py +``` + +项目目录中包含一个`__pypackages__`目录，用来放置所有依赖的库文件，就像`npm`的`node_modules`一样。 +你可以在[这里](https://www.python.org/dev/peps/pep-0582/#specification)阅读更多提案的细节。 + ## 安装 PDM 需要 Python 3.7 或更高版本。 @@ -148,7 +148,7 @@ pdm init 按照指引回答提示的问题，一个 PDM 项目和对应的`pyproject.toml`文件就创建好了。 -**把依赖安装到 `__pypackages__` 文件夹中** +**添加依赖** ```bash pdm add requests flask @@ -158,6 +158,12 @@ pdm add requests flask **在 [PEP 582] 加持下运行你的脚本** +默认情况下，当你在一个项目中第一次运行 `pdm install`, PDM 会为你在项目根目录的 `.venv` 中创建一个虚拟环境，和其他包管理器一样。 +但你也可以把 PEP 582 设为默认，只需要运行 `pdm config python.use_venv false` 就可以了。除此之外，你还需要一点点的配置，让 Python 解释器 +可以用 PEP 582 的 `__papackages__` 目录来查找包。 + +````bash + 假设你在`__pypackages__`同级的目录下有一个`app.py`脚本，内容如下（从 Flask 的官网例子复制而来）： ```python @@ -170,7 +176,7 @@ def hello_world(): if __name__ == '__main__': app.run() -``` +```` 如果你使用的是 Bash，可以通过执行`eval "$(pdm --pep582)"`设置环境变量，现在你可以用你最熟悉的 **Python 解释器** 运行脚本了： diff --git a/docs/docs/index.md b/docs/docs/index.md index 1d87203c..23bc2bdc 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -11,7 +11,7 @@ in a similar way to `npm` that doesn't need to create a virtualenv at all! ## Feature highlights -- [PEP 582] local package installer and runner, no virtualenv involved at all. +- Opt-in [PEP 582] support, no virtualenv involved at all. - Simple and fast dependency resolver, mainly for large binary distributions. - A [PEP 517] build backend. - [PEP 621] project metadata. @@ -132,30 +132,6 @@ You can either pass the options after the script or set the env var value. .\pw --init pdm ``` -### Enable PEP 582 globally - -To make the Python interpreters aware of PEP 582 packages, one need to add the `pdm/pep582/sitecustomize.py` -to the Python library search path. - -#### For Windows users - -One just needs to execute `pdm --pep582`, then environment variable will be changed automatically. Don't forget -to restart the terminal session to take effect. - -#### For Mac and Linux users - -The command to change the environment variables can be printed by `pdm --pep582 []`. If `` -isn't given, PDM will pick one based on some guesses. You can run `eval "$(pdm --pep582)"` to execute the command. - -You may want to write a line in your `.bash_profile`(or similar profiles) to make it effective when logging in. -For example, in bash you can do this: - -```bash -pdm --pep582 >> ~/.bash_profile -``` - -Once again, Don't forget to restart the terminal session to take effect. - ## Shell Completion PDM supports generating completion scripts for Bash, Zsh, Fish or Powershell. Here are some common locations for each shell: @@ -203,129 +179,10 @@ PDM supports generating completion scripts for Bash, Zsh, Fish or Powershell. He pdm completion powershell | Out-File -Encoding utf8 $PROFILE\..\Completions\pdm_completion.ps1 ``` -## Use with IDE - -Now there are not built-in support or plugins for PEP 582 in most IDEs, you have to configure your tools manually. - -PDM will write and store project-wide configurations in `.pdm.toml` and you are recommended to add following lines -in the `.gitignore`: - -``` -.pdm.toml -__pypackages__/ -``` - -### PyCharm - -Mark `__pypackages__//lib` as [Sources Root](https://www.jetbrains.com/help/pycharm/configuring-project-structure.html#mark-dir-project-view). -Then, select as [Python interpreter](https://www.jetbrains.com/help/pycharm/configuring-python-interpreter.html#interpreter) a Python installation with the same `` version. - -Additionally, if you want to use tools from the environment (e.g. `pytest`), you have to add the -`__pypackages__//bin` directory to the `PATH` variable in the corresponding -run/debug configuration. - -### VSCode - -Add the following two entries to the top-level dict in `.vscode/settings.json`: - -```json -{ - "python.autoComplete.extraPaths": ["__pypackages__//lib"], - "python.analysis.extraPaths": ["__pypackages__//lib"] -} -``` - -[Enable PEP582 globally](#enable-pep-582-globally), -and make sure VSCode runs using the same user and shell you enabled PEP582 for. - -??? note "Cannot enable PEP582 globally?" - If for some reason you cannot enable PEP582 globally, you can still configure each "launch" in each project: - set the `PYTHONPATH` environment variable in your launch configuration, in `.vscode/launch.json`. - For example, to debug your `pytest` run: - - ```json - { - "version": "0.2.0", - "configurations": [ - { - "name": "pytest", - "type": "python", - "request": "launch", - "module": "pytest", - "args": ["tests"], - "justMyCode": false, - "env": {"PYTHONPATH": "__pypackages__//lib"} - } - ] - } - ``` - - If your package resides in a `src` directory, add it to `PYTHONPATH` as well: - - ```json - "env": {"PYTHONPATH": "src:__pypackages__//lib"} - ``` - -??? note "Using Pylance/Pyright?" - If you have configured `"python.analysis.diagnosticMode": "workspace"`, - and you see a ton of errors/warnings as a result. - you may need to create `pyrightconfig.json` in the workspace directory, and fill in the following fields: - - ```json - { - "exclude": ["__pypackages__"] - } - ``` - - Then restart the language server or VS Code and you're good to go. - In the future ([microsoft/pylance-release#1150](https://github.com/microsoft/pylance-release/issues/1150)), maybe the problem will be solved. - -??? note "Using Jupyter Notebook?" - If you wish to use pdm to install jupyter notebook and use it in vscode in conjunction with the python extension: - - 1. Use `pdm add notebook` or so to install notebook - 2. Add a `.env` file inside of your project director with contents like the following: - - ``` - PYTHONPATH=/your-workspace-path/__pypackages__/./lib - ``` - - If the above still doesn't work, it's most likely because the environment variable is not properly loaded when the Notebook starts. There are two workarounds. - - 1. Run `code .` in Terminal. It will open a new VSCode window in the current directory with the path set correctly. Use the Jupyter Notebook in the new window - 2. If you prefer not to open a new window, run the following at the beginning of your Jupyter Notebook to explicitly set the path: - - ``` - import sys - sys.path.append('/your-workspace-path/__pypackages__/./lib') - ``` - - > [Reference Issue](https://github.com/pdm-project/pdm/issues/848) - -#### Task Provider - -In addition, there is a [VSCode Task Provider extension][pdm task provider] available for download. - -This makes it possible for VSCode to automatically detect [pdm scripts][pdm scripts] so they -can be run natively as [VSCode Tasks][vscode tasks]. - -[vscode tasks]: https://code.visualstudio.com/docs/editor/tasks -[pdm task provider]: https://marketplace.visualstudio.com/items?itemName=knowsuchagency.pdm-task-provider -[pdm scripts]: usage/scripts.md - -### Neovim - -If using [neovim-lsp](https://github.com/neovim/nvim-lspconfig) with -[pyright](https://github.com/Microsoft/pyright) and want your -`__pypackages__` directory to be added to the path, you can add this to your -project's `pyproject.toml`. - -```toml -[tool.pyright] -extraPaths = ["__pypackages__//lib/"] -``` +## Virtualenv and PEP 582 -### [Seek for other IDEs or editors](usage/advanced.md#integrate-with-other-ide-or-editors) +In addition to the virtualenv management, PDM supports [PEP 582](https://www.python.org/dev/peps/pep-0582/) as an opt-in feature. +You can learn more about the two modes in the corresponding chapters in [Working with virtualenv](usage/venv.md) and [Working with PEP 582](usage/pep582.md) ## PDM Eco-system diff --git a/docs/docs/plugin/write.md b/docs/docs/plugin/write.md index 4a9c061e..0f36fff6 100644 --- a/docs/docs/plugin/write.md +++ b/docs/docs/plugin/write.md @@ -129,7 +129,7 @@ by `pip install -e .` or `python setup.py develop` in the **traditional** Python as there is no such `setup.py` in a PDM project, how can we do that? Fortunately, it becomes even easier with PDM and PEP 582. First, you should enable PEP 582 globally following the -[corresponding part of this doc](../index.md#enable-pep-582-globally). Then you just need to install all dependencies into the `__pypackages__` directory by: +[corresponding part of this doc](../usage/pep582.md#enable-pep-582-globally). Then you just need to install all dependencies into the `__pypackages__` directory by: ```bash pdm install diff --git a/docs/docs/usage/configuration.md b/docs/docs/usage/configuration.md index 1982735e..c31ffe05 100644 --- a/docs/docs/usage/configuration.md +++ b/docs/docs/usage/configuration.md @@ -1,28 +1,31 @@ ## Available Configurations -The following configuration items can be retrieved and modified by [`pdm config`](usage/cli_reference.md#exec-0--config) command. +The following configuration items can be retrieved and modified by [`pdm config`](../usage/cli_reference.md#exec-0--config) command. -| Config Item | Description | Default Value | Available in Project | Env var | -| ----------------------------- | ------------------------------------------------------------------------- | ------------------------------------------------------------------------- | -------------------- | ------------------------ | -| `build_isolation` | Isolate the build environment from the project environment | Yes | True | `PDM_BUILD_ISOLATION` | -| `cache_dir` | The root directory of cached files | The default cache location on OS | No | | -| `check_update` | Check if there is any newer version available | True | No | | -| `global_project.fallback` | Use the global project implicitly if no local project is found | `False` | No | | -| `global_project.fallback_verbose` | If True show message when global project is used implicitly | `True` | No | | -| `global_project.path` | The path to the global project | `/global-project` | No | | -| `global_project.user_site` | Whether to install to user site | `False` | No | | -| `install.cache` | Enable caching of wheel installations | False | Yes | | -| `install.cache_method` | Specify how to create links to the caches(`symlink` or `pth`) | `symlink` | Yes | | -| `install.parallel` | Whether to perform installation and uninstallation in parallel | `True` | Yes | `PDM_PARALLEL_INSTALL` | -| `project_max_depth` | The max depth to search for a project through the parents | 5 | No | `PDM_PROJECT_MAX_DEPTH` | -| `python.path` | The Python interpreter path | | Yes | `PDM_PYTHON` | -| `python.use_pyenv` | Use the pyenv interpreter | `True` | Yes | | -| `python.use_venv` | Install packages into the activated venv site packages instead of PEP 582 | `False` | Yes | `PDM_USE_VENV` | -| `pypi.url` | The URL of PyPI mirror | Read `index-url` in `pip.conf`, or `https://pypi.org/simple` if not found | Yes | `PDM_PYPI_URL` | -| `pypi.verify_ssl` | Verify SSL certificate when query PyPI | Read `trusted-hosts` in `pip.conf`, defaults to `True` | Yes | | -| `pypi.json_api` | Consult PyPI's JSON API for package metadata | `False` | Yes | `PDM_PYPI_JSON_API` | -| `strategy.save` | Specify how to save versions when a package is added | `compatible`(can be: `exact`, `wildcard`, `minimum`) | Yes | | -| `strategy.update` | The default strategy for updating packages | `reuse`(can be : `eager`) | Yes | | -| `strategy.resolve_max_rounds` | Specify the max rounds of resolution process | 1000 | Yes | `PDM_RESOLVE_MAX_ROUNDS` | +| Config Item | Description | Default Value | Available in Project | Env var | +| --------------------------------- | ------------------------------------------------------------------------- | ------------------------------------------------------------------------- | -------------------- | ------------------------ | +| `build_isolation` | Isolate the build environment from the project environment | Yes | True | `PDM_BUILD_ISOLATION` | +| `cache_dir` | The root directory of cached files | The default cache location on OS | No | | +| `check_update` | Check if there is any newer version available | True | No | | +| `global_project.fallback` | Use the global project implicitly if no local project is found | `False` | No | | +| `global_project.fallback_verbose` | If True show message when global project is used implicitly | `True` | No | | +| `global_project.path` | The path to the global project | `/global-project` | No | | +| `global_project.user_site` | Whether to install to user site | `False` | No | | +| `install.cache` | Enable caching of wheel installations | False | Yes | | +| `install.cache_method` | Specify how to create links to the caches(`symlink` or `pth`) | `symlink` | Yes | | +| `install.parallel` | Whether to perform installation and uninstallation in parallel | `True` | Yes | `PDM_PARALLEL_INSTALL` | +| `project_max_depth` | The max depth to search for a project through the parents | 5 | No | `PDM_PROJECT_MAX_DEPTH` | +| `python.path` | The Python interpreter path | | Yes | `PDM_PYTHON` | +| `python.use_pyenv` | Use the pyenv interpreter | `True` | Yes | | +| `python.use_venv` | Install packages into the activated venv site packages instead of PEP 582 | `False` | Yes | `PDM_USE_VENV` | +| `pypi.url` | The URL of PyPI mirror | Read `index-url` in `pip.conf`, or `https://pypi.org/simple` if not found | Yes | `PDM_PYPI_URL` | +| `pypi.verify_ssl` | Verify SSL certificate when query PyPI | Read `trusted-hosts` in `pip.conf`, defaults to `True` | Yes | | +| `pypi.json_api` | Consult PyPI's JSON API for package metadata | `False` | Yes | `PDM_PYPI_JSON_API` | +| `strategy.save` | Specify how to save versions when a package is added | `compatible`(can be: `exact`, `wildcard`, `minimum`) | Yes | | +| `strategy.update` | The default strategy for updating packages | `reuse`(can be : `eager`) | Yes | | +| `strategy.resolve_max_rounds` | Specify the max rounds of resolution process | 1000 | Yes | `PDM_RESOLVE_MAX_ROUNDS` | +| `venv.location` | Parent directory for virtualenvs | `/venvs` | No | | +| `venv.backend` | Default backend to create virtualenv | `virtualenv` | Yes | `PDM_VENV_BACKEND` | +| `venv.in-project` | Create virtualenv in `.venv` under project root | `False` | Yes | `PDM_VENV_IN_PROJECT` | _If the corresponding env var is set, the value will take precedence over what is saved in the config file._ diff --git a/docs/docs/usage/pep582.md b/docs/docs/usage/pep582.md new file mode 100644 index 00000000..cccbbae2 --- /dev/null +++ b/docs/docs/usage/pep582.md @@ -0,0 +1,162 @@ +# Working with PEP 582 + + +With [PEP 582](https://www.python.org/dev/peps/pep-0582/), dependencies will be installed into `__pypackages__` directory under the project root. With [PEP 582 enabled globally](#enable-pep-582-globally), you can also use the project interpreter to run scripts directly. + +**When the project interpreter is a normal Python, this mode is enabled.** + +Besides, on a project you work with for the first time on your machine, if it contains an empty `__pypackages__` directory, PEP 582 is enabled automatically, and virtualenv won't be created. + +## Enable PEP 582 globally + +To make the Python interpreters aware of PEP 582 packages, one need to add the `pdm/pep582/sitecustomize.py` +to the Python library search path. + +=== "Windows" + + One just needs to execute `pdm --pep582`, then environment variable will be changed automatically. Don't forget + to restart the terminal session to take effect. + +=== "Mac and Linux" + + The command to change the environment variables can be printed by `pdm --pep582 []`. If `` + isn't given, PDM will pick one based on some guesses. You can run `eval "$(pdm --pep582)"` to execute the command. + + You may want to write a line in your `.bash_profile`(or similar profiles) to make it effective when logging in. + For example, in bash you can do this: + + ```bash + pdm --pep582 >> ~/.bash_profile + ``` + + Once again, Don't forget to restart the terminal session to take effect. + +??? note "How is it done?" + + Thanks to the [site packages loading](https://docs.python.org/3/library/site.html) on Python startup. It is possible to patch the `sys.path` + by executing the `sitecustomize.py` shipped with PDM. The interpreter can search the directories + for the nearest `__pypackage__` folder and append it to the `sys.path` variable. + +## Configure IDE to support PEP 582 + +Now there are no built-in support or plugins for PEP 582 in most IDEs, you have to configure your tools manually. + +PDM will write and store project-wide configurations in `.pdm.toml` and you are recommended to add following lines +in the `.gitignore`: + +``` +.pdm.toml +__pypackages__/ +``` + +### PyCharm + +Mark `__pypackages__//lib` as [Sources Root](https://www.jetbrains.com/help/pycharm/configuring-project-structure.html#mark-dir-project-view). +Then, select as [Python interpreter](https://www.jetbrains.com/help/pycharm/configuring-python-interpreter.html#interpreter) a Python installation with the same `` version. + +Additionally, if you want to use tools from the environment (e.g. `pytest`), you have to add the +`__pypackages__//bin` directory to the `PATH` variable in the corresponding +run/debug configuration. + +### VSCode + +Add the following two entries to the top-level dict in `.vscode/settings.json`: + +```json +{ + "python.autoComplete.extraPaths": ["__pypackages__//lib"], + "python.analysis.extraPaths": ["__pypackages__//lib"] +} +``` + +[Enable PEP582 globally](#enable-pep-582-globally), +and make sure VSCode runs using the same user and shell you enabled PEP582 for. + +??? note "Cannot enable PEP582 globally?" + If for some reason you cannot enable PEP582 globally, you can still configure each "launch" in each project: + set the `PYTHONPATH` environment variable in your launch configuration, in `.vscode/launch.json`. + For example, to debug your `pytest` run: + + ```json + { + "version": "0.2.0", + "configurations": [ + { + "name": "pytest", + "type": "python", + "request": "launch", + "module": "pytest", + "args": ["tests"], + "justMyCode": false, + "env": {"PYTHONPATH": "__pypackages__//lib"} + } + ] + } + ``` + + If your package resides in a `src` directory, add it to `PYTHONPATH` as well: + + ```json + "env": {"PYTHONPATH": "src:__pypackages__//lib"} + ``` + +??? note "Using Pylance/Pyright?" + If you have configured `"python.analysis.diagnosticMode": "workspace"`, + and you see a ton of errors/warnings as a result. + you may need to create `pyrightconfig.json` in the workspace directory, and fill in the following fields: + + ```json + { + "exclude": ["__pypackages__"] + } + ``` + + Then restart the language server or VS Code and you're good to go. + In the future ([microsoft/pylance-release#1150](https://github.com/microsoft/pylance-release/issues/1150)), maybe the problem will be solved. + +??? note "Using Jupyter Notebook?" + If you wish to use pdm to install jupyter notebook and use it in vscode in conjunction with the python extension: + + 1. Use `pdm add notebook` or so to install notebook + 2. Add a `.env` file inside of your project director with contents like the following: + + ``` + PYTHONPATH=/your-workspace-path/__pypackages__/./lib + ``` + + If the above still doesn't work, it's most likely because the environment variable is not properly loaded when the Notebook starts. There are two workarounds. + + 1. Run `code .` in Terminal. It will open a new VSCode window in the current directory with the path set correctly. Use the Jupyter Notebook in the new window + 2. If you prefer not to open a new window, run the following at the beginning of your Jupyter Notebook to explicitly set the path: + + ``` + import sys + sys.path.append('/your-workspace-path/__pypackages__/./lib') + ``` + + > [Reference Issue](https://github.com/pdm-project/pdm/issues/848) + +??? note "PDM Task Provider" + + In addition, there is a [VSCode Task Provider extension][pdm task provider] available for download. + + This makes it possible for VSCode to automatically detect [pdm scripts][pdm scripts] so they + can be run natively as [VSCode Tasks][vscode tasks]. + + [vscode tasks]: https://code.visualstudio.com/docs/editor/tasks + [pdm task provider]: https://marketplace.visualstudio.com/items?itemName=knowsuchagency.pdm-task-provider + [pdm scripts]: scripts.md + +### Neovim + +If using [neovim-lsp](https://github.com/neovim/nvim-lspconfig) with +[pyright](https://github.com/Microsoft/pyright) and want your +`__pypackages__` directory to be added to the path, you can add this to your +project's `pyproject.toml`. + +```toml +[tool.pyright] +extraPaths = ["__pypackages__//lib/"] +``` + +### [Seek for other IDEs or editors](advanced.md#integrate-with-other-ide-or-editors) diff --git a/docs/docs/usage/project.md b/docs/docs/usage/project.md index f7f7f2a1..090c6d87 100644 --- a/docs/docs/usage/project.md +++ b/docs/docs/usage/project.md @@ -5,7 +5,7 @@ If you have used [`pdm init`](cli_reference.md#exec-0--init), you must have already seen how PDM detects and selects the Python interpreter. After initialized, you can also change the settings by `pdm use `. The argument can be either a version specifier of any length, or a relative or absolute path to the -python interpreter, but remember the Python interpreter must conform with the `python_requires` +python interpreter, but remember the Python interpreter must conform with the `requires-python` constraint in the project file. ### How `requires-python` controls the project @@ -159,9 +159,16 @@ The caches are located under `$(pdm config cache_dir)/packages`. One can view th ```bash $ pdm info -Python Interpreter: D:/Programs/Python/Python38/python.exe (3.8.0) -Project Root: D:/Workspace/pdm - [10:42] +PDM version: + 2.0.0 +Python Interpreter: + /opt/homebrew/opt/python@3.9/bin/python3.9 (3.9) +Project Root: + /Users/fming/wkspace/github/test-pdm +Project Packages: + /Users/fming/wkspace/github/test-pdm/__pypackages__/3.9 + +# Show environment info $ pdm info --env { "implementation_name": "cpython", @@ -178,6 +185,11 @@ $ pdm info --env } ``` +[This command](cli_reference.md#exec-0--info) is useful for checking which mode is being used by the project: + +- If *Project Packages* is `None`, [virtualenv mode](venv.md) is enabled. +- Otherwise, [PEP 582 mode](pep582.md) is enabled. + ## Manage global project Sometimes users may want to keep track of the dependencies of global Python interpreter as well. @@ -199,19 +211,6 @@ project path via `-p/--project ` option. !!! attention "CAUTION" Be careful with `remove` and `sync --clean/--pure` commands when global project is used, because it may remove packages installed in your system Python. -## Working with a virtualenv - -Although PDM enforces PEP 582 by default, it also allows users to install packages into the virtualenv. It is controlled -by the configuration item `python.use_venv`. When it is set to `True`, PDM will use the virtualenv if: - -- a virtualenv is already activated. -- any of `venv`, `.venv`, `env` is a valid virtualenv folder. - -Besides, when `python.use_venv` is on and the interpreter path given is a venv-like path, PDM will reuse that venv directory as well. - -For enhanced virtualenv support such as virtualenv management and auto-creation, please go for [pdm-venv](https://github.com/pdm-project/pdm-venv), -which can be installed as a plugin. - ## Import project metadata from existing project files If you are already other package manager tools like Pipenv or Poetry, it is easy to migrate to PDM. @@ -279,9 +278,3 @@ PDM provides a convenient command group to manage the cache, there are four kind See the current cache usage by typing `pdm cache info`. Besides, you can use `add`, `remove` and `list` subcommands to manage the cache content. Find the usage by the `--help` option of each command. - -## How we make PEP 582 packages available to the Python interpreter - -Thanks to the [site packages loading](https://docs.python.org/3/library/site.html) on Python startup. It is possible to patch the `sys.path` -by executing the `sitecustomize.py` shipped with PDM. The interpreter can search the directories -for the nearest `__pypackage__` folder and append it to the `sys.path` variable. diff --git a/docs/docs/usage/venv.md b/docs/docs/usage/venv.md new file mode 100644 index 00000000..cc66f879 --- /dev/null +++ b/docs/docs/usage/venv.md @@ -0,0 +1,106 @@ +# Working with virtualenv + +When you run [`pdm init`](cli_reference.md#exec-0--init) command, PDM will [ask for the Python interpreter to use](project.md#choose-a-python-interpreter) in the project, which is the base interpreter to install dependencies and run tasks. + +Compared to [PEP 582](https://www.python.org/dev/peps/pep-0582/), virtual environments are considered more mature and have better support in the Python ecosystem as well as IDEs. Therefore, virtualenv is the default mode if not configured otherwise. + +**Virtual environments will be used if the project interpreter(the interpreter stored in `.pdm.toml`, which can be checked by `pdm info`) is from a virtualenv.** + +## Virtualenv auto-creation + +By default, PDM prefers to use the virtualenv layout as other package managers do. When you run `pdm install` the first time on a new PDM-managed project, whose Python interpreter is not decided yet, PDM will create a virtualenv in `/.venv`, and install dependencies into it. In the interactive session of `pdm init`, PDM will also ask to create a virtualenv for you. + +You can choose the backend used by PDM to create a virtualenv. Currently it supports three backends: + +- [`virtualenv`](https://virtualenv.pypa.io/)(default) +- `venv` +- `conda` + +You can change it by `pdm config venv.backend [virtualenv|venv|conda]`. + +## Create a virtualenv yourself + +You can create more than one virtualenvs with whatever Python version you want. + +```bash +# Create a virtualenv based on 3.8 interpreter +$ pdm venv create 3.8 +# Assign a different name other than the version string +$ pdm venv create --name for-test 3.8 +# Use venv as the backend to create, support 3 backends: virtualenv(default), venv, conda +$ pdm venv create --with venv 3.9 +``` + +## The location of virtualenvs + +For the first time, PDM will try to create a virtualenv **in project**, unless `.venv` already exists. +Other virtualenvs go to the location specified by the `venv.location` configuration. They are named as `--` to avoid name collision. A virtualenv created with `--name` option will always go to this location. You can disable the in-project virtualenv creation by `pdm config venv.in_project false`. + +## Virtualenv auto-detection + +When no interpreter is stored in the project config or `PDM_IGNORE_SAVED_PYTHON` env var is set, PDM will try to detect possible virtualenvs to use: + +- `venv`, `env`, `.venv` directories in the project root +- The currently activated virtualenv + +## List all virtualenvs created with this project + +```bash +$ pdm venv list +Virtualenvs created with this project: + +- 3.8.6: C:\Users\Frost Ming\AppData\Local\pdm\pdm\venvs\test-project-8Sgn_62n-3.8.6 +- for-test: C:\Users\Frost Ming\AppData\Local\pdm\pdm\venvs\test-project-8Sgn_62n-for-test +- 3.9.1: C:\Users\Frost Ming\AppData\Local\pdm\pdm\venvs\test-project-8Sgn_62n-3.9.1 +``` + +## Remove a virtualenv + +```bash +$ pdm venv remove for-test +Virtualenvs created with this project: +Will remove: C:\Users\Frost Ming\AppData\Local\pdm\pdm\venvs\test-project-8Sgn_62n-for-test, continue? [y/N]:y +Removed C:\Users\Frost Ming\AppData\Local\pdm\pdm\venvs\test-project-8Sgn_62n-for-test +``` + +## Activate a virtualenv + +Instead of spawning a subshell like what `pipenv` and `poetry` do, `pdm-venv` doesn't create the shell for you but print the activate command to the console. In this way you won't leave the current shell. You can then feed the output to `eval` to activate the virtualenv: + +=== "bash/csh/zsh" + + ```bash + $ eval $(pdm venv activate for-test) + (test-project-8Sgn_62n-for-test) $ # Virtualenv entered + Fish + + $ eval (pdm venv activate for-test) + ``` + +=== "Powershell" + + ```ps1 + PS1> Invoke-Expression (pdm venv activate for-test) + ``` + + You can make your own shell shortcut function to avoid the input of long command. Here is an example of Bash: + + ```ps1 + pdm_venv_activate() { + eval $('pdm' 'venv' 'activate' "$1") + } + ``` + + Then you can activate it by `pdm_venv_activate $venv_name` and deactivate by deactivate directly. + + Additionally, if the project interpreter is a venv Python, you can omit the name argument following activate. + +!!! NOTE + `venv activate` **does not** switch the Python interpreter used by the project. It only changes the shell by injecting the virtualenv paths to environment variables. For the forementioned purpose, use the `pdm use` command. + +For more CLI usage, see the [`pdm venv`](cli_reference.md#exec-0--venv) documentation. + +## Disable virtualenv mode + +You can disable the auto-creation and auto-detection for virtualenv by `pdm config python.use_venv` false. +**If venv is disabled, PEP 582 mode will always be used even if the selected interpreter is from a virtualenv.** diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 617d8cf2..1ca8995b 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -41,8 +41,10 @@ plugins: nav: - Usage: - Home: index.md - - usage/dependency.md - usage/project.md + - usage/venv.md + - usage/pep582.md + - usage/dependency.md - usage/scripts.md - usage/hooks.md - usage/advanced.md @@ -76,7 +78,7 @@ markdown_extensions: custom_fences: - name: mermaid class: mermaid - format: !!python/name:pymdownx.superfences.fence_code_format + format: "!!python/name:pymdownx.superfences.fence_code_format" copyright: Copyright © 2019-2021 Frost Ming diff --git a/news/1162.feature.md b/news/1162.feature.md new file mode 100644 index 00000000..da5991e2 --- /dev/null +++ b/news/1162.feature.md @@ -0,0 +1 @@ +Integrate `pdm venv` commands into the main program. Make PEP 582 an opt-in feature. diff --git a/news/1191.removal.md b/news/1191.removal.md new file mode 100644 index 00000000..2517cae0 --- /dev/null +++ b/news/1191.removal.md @@ -0,0 +1 @@ +Remove the useless `--no-clean` option from `pdm sync` command. diff --git a/pdm.lock b/pdm.lock index c7881c4b..088e9f80 100644 --- a/pdm.lock +++ b/pdm.lock @@ -132,7 +132,7 @@ summary = "A platform independent file lock." [[package]] name = "findpython" -version = "0.1.6" +version = "0.2.0" requires_python = ">=3.7" summary = "A utility to find python versions on your system" dependencies = [ @@ -711,7 +711,7 @@ summary = "Backport of pathlib-compatible object wrapper for zip files" [metadata] lock_version = "3.1" -content_hash = "sha256:849f0d5912f6dc46efa4f7919c586cae25c43c3dc5ff8939a3e7ef0f0497e237" +content_hash = "sha256:0f7b10222fac8f4982fd1632d5ee9385d3e9fc6ffaa4317cdd1fb965627ce585" [metadata.files] "arpeggio 1.10.2" = [ @@ -819,9 +819,9 @@ content_hash = "sha256:849f0d5912f6dc46efa4f7919c586cae25c43c3dc5ff8939a3e7ef0f0 {file = "filelock-3.7.1-py3-none-any.whl", hash = "sha256:37def7b658813cda163b56fc564cdc75e86d338246458c4c28ae84cabefa2404"}, {file = "filelock-3.7.1.tar.gz", hash = "sha256:3a0fd85166ad9dbab54c9aec96737b744106dc5f15c0b09a6744a445299fcf04"}, ] -"findpython 0.1.6" = [ - {file = "findpython-0.1.6-py3-none-any.whl", hash = "sha256:79ec09965019b73c83f49df0de9a056d2055abf694f3813966a4841e80cf734e"}, - {file = "findpython-0.1.6.tar.gz", hash = "sha256:9fd6185cdcb96baa7109308447efb493b2c7f1a8f569e128af14d726b2a69e18"}, +"findpython 0.2.0" = [ + {file = "findpython-0.2.0-py3-none-any.whl", hash = "sha256:110ec222a43aca3fcd154fd90b911f465c70e86787ae0532bab2266a95870fc9"}, + {file = "findpython-0.2.0.tar.gz", hash = "sha256:c2099ee0b71fc2714b64f68fd1f40bc0ee47f49dfe9547fb64d7cbcc02fe0871"}, ] "ghp-import 2.1.0" = [ {file = "ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343"}, diff --git a/pdm/cli/actions.py b/pdm/cli/actions.py index ec03f622..0850051f 100644 --- a/pdm/cli/actions.py +++ b/pdm/cli/actions.py @@ -590,8 +590,9 @@ def do_use( python: str = "", first: bool = False, ignore_remembered: bool = False, + ignore_requires_python: bool = False, hooks: HookManager | None = None, -) -> None: +) -> PythonInfo: """Use the specified python version and save in project config. The python can be a version string or interpreter path. """ @@ -601,7 +602,9 @@ def do_use( python = python.strip() def version_matcher(py_version: PythonInfo) -> bool: - return project.python_requires.contains(str(py_version.version), True) + return ignore_requires_python or project.python_requires.contains( + str(py_version.version), True + ) if not project.cache_dir.exists(): project.cache_dir.mkdir(parents=True) @@ -681,6 +684,7 @@ def do_use( project.core.ui.echo("Updating executable scripts...", style="cyan") project.environment.update_shebangs(selected_python.executable.as_posix()) hooks.try_emit("post_use", python=selected_python) + return selected_python def do_import( diff --git a/pdm/cli/commands/init.py b/pdm/cli/commands/init.py index 5293b454..7dc8eca0 100644 --- a/pdm/cli/commands/init.py +++ b/pdm/cli/commands/init.py @@ -5,8 +5,9 @@ from pdm.cli import actions from pdm.cli.commands.base import BaseCommand from pdm.cli.hooks import HookManager from pdm.cli.options import skip_option +from pdm.models.python import PythonInfo from pdm.project import Project -from pdm.utils import get_user_email_from_git +from pdm.utils import get_user_email_from_git, get_venv_like_prefix class Command(BaseCommand): @@ -35,6 +36,8 @@ class Command(BaseCommand): parser.set_defaults(search_parent=False) def handle(self, project: Project, options: argparse.Namespace) -> None: + from pdm.cli.commands.venv.utils import get_venv_python + hooks = HookManager(project, options.skip) if project.pyproject_file.exists(): project.core.ui.echo( @@ -45,13 +48,36 @@ class Command(BaseCommand): self.set_interactive(not options.non_interactive) if self.interactive: - actions.do_use(project, hooks=hooks) + python = actions.do_use(project, ignore_requires_python=True, hooks=hooks) + if ( + project.config["python.use_venv"] + and get_venv_like_prefix(python.executable) is None + ): + if termui.confirm( + "Would you like to create a virtualenv with " + f"[green]{python.executable}[/]?", + default=True, + ): + try: + path = project._create_virtualenv() + project.python = PythonInfo.from_path(get_venv_python(path)) + except Exception as e: # pragma: no cover + project.core.ui.echo( + f"Error occured when creating virtualenv: {e}\n" + "Please fix it and create later.", + style="red", + err=True, + ) else: - actions.do_use(project, "3", True, hooks=hooks) - is_library = ( - termui.confirm( - "Is the project a library that will be uploaded to PyPI", default=False + actions.do_use(project, "3", True, ignore_requires_python=True, hooks=hooks) + if get_venv_like_prefix(project.python.executable) is None: + project.core.ui.echo( + "You are using the PEP 582 mode, no virtualenv is created.\n" + "For more info, please visit https://peps.python.org/pep-0582/", + style="green", ) + is_library = ( + termui.confirm("Is the project a library that will be uploaded to PyPI") if self.interactive else False ) diff --git a/pdm/cli/commands/plugin.py b/pdm/cli/commands/plugin.py index ef785cac..b6b94fdc 100644 --- a/pdm/cli/commands/plugin.py +++ b/pdm/cli/commands/plugin.py @@ -166,7 +166,9 @@ class RemoveCommand(BaseCommand): sys.exit(1) if not ( options.yes - or termui.confirm(f"Will remove: {packages_to_remove}, continue?") + or termui.confirm( + f"Will remove: {packages_to_remove}, continue?", default=True + ) ): return pip_args = ( diff --git a/pdm/cli/commands/venv/__init__.py b/pdm/cli/commands/venv/__init__.py new file mode 100644 index 00000000..afc17723 --- /dev/null +++ b/pdm/cli/commands/venv/__init__.py @@ -0,0 +1,30 @@ +import argparse +from typing import List + +from pdm import Project +from pdm.cli.commands.base import BaseCommand +from pdm.cli.commands.venv.activate import ActivateCommand +from pdm.cli.commands.venv.create import CreateCommand +from pdm.cli.commands.venv.list import ListCommand +from pdm.cli.commands.venv.purge import PurgeCommand +from pdm.cli.commands.venv.remove import RemoveCommand +from pdm.cli.options import Option + + +class Command(BaseCommand): + """Virtualenv management""" + + name = "venv" + arguments: List[Option] = [] + + def add_arguments(self, parser: argparse.ArgumentParser) -> None: + subparser = parser.add_subparsers() + CreateCommand.register_to(subparser, "create") + ListCommand.register_to(subparser, "list") + RemoveCommand.register_to(subparser, "remove") + ActivateCommand.register_to(subparser, "activate") + PurgeCommand.register_to(subparser, "purge") + self.parser = parser + + def handle(self, project: Project, options: argparse.Namespace) -> None: + self.parser.print_help() diff --git a/pdm/cli/commands/venv/activate.py b/pdm/cli/commands/venv/activate.py new file mode 100644 index 00000000..109d69ac --- /dev/null +++ b/pdm/cli/commands/venv/activate.py @@ -0,0 +1,62 @@ +import argparse +import shlex +from pathlib import Path + +import shellingham + +from pdm.cli.commands.base import BaseCommand +from pdm.cli.commands.venv.utils import BIN_DIR, iter_venvs +from pdm.cli.options import verbose_option +from pdm.project import Project +from pdm.utils import get_venv_like_prefix + + +class ActivateCommand(BaseCommand): + """Activate the virtualenv with the given name""" + + arguments = [verbose_option] + + def add_arguments(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument("env", nargs="?", help="The key of the virtualenv") + + def handle(self, project: Project, options: argparse.Namespace) -> None: + if options.env: + venv = next( + (venv for key, venv in iter_venvs(project) if key == options.env), None + ) + if not venv: + project.core.ui.echo( + f"No virtualenv with key [green]{options.env}[/] is found", + style="yellow", + err=True, + ) + raise SystemExit(1) + else: + # Use what is saved in .pdm.toml + interpreter = project.python_executable + venv = get_venv_like_prefix(interpreter) + if venv is None: + project.core.ui.echo( + f"Can't activate a non-venv Python [green]{interpreter}[/], " + "you can specify one with [green]pdm venv activate [/]", + style="yellow", + err=True, + ) + raise SystemExit(1) + project.core.ui.echo(self.get_activate_command(venv)) + + def get_activate_command(self, venv: Path) -> str: # pragma: no cover + shell, _ = shellingham.detect_shell() + if shell == "fish": + command, filename = "source", "activate.fish" + elif shell == "csh": + command, filename = "source", "activate.csh" + elif shell in ["powershell", "pwsh"]: + command, filename = ".", "Activate.ps1" + else: + command, filename = "source", "activate" + activate_script = venv / BIN_DIR / filename + if activate_script.exists(): + return f"{command} {shlex.quote(str(activate_script))}" + # Conda backed virtualenvs don't have activate scripts + return f"conda activate {shlex.quote(str(venv))}" diff --git a/pdm/cli/commands/venv/backends.py b/pdm/cli/commands/venv/backends.py new file mode 100644 index 00000000..62e1d828 --- /dev/null +++ b/pdm/cli/commands/venv/backends.py @@ -0,0 +1,158 @@ +import abc +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any, List, Mapping, Optional, Tuple, Type + +from pdm import Project, termui +from pdm.cli.commands.venv.utils import get_venv_prefix +from pdm.exceptions import PdmUsageError, ProjectError +from pdm.models.python import PythonInfo +from pdm.utils import cached_property + + +class VirtualenvCreateError(ProjectError): + pass + + +class Backend(abc.ABC): + """The base class for virtualenv backends""" + + def __init__(self, project: Project, python: Optional[str]) -> None: + self.project = project + self.python = python + + @cached_property + def _resolved_interpreter(self) -> PythonInfo: + if not self.python: + return self.project.python + try: # pragma: no cover + return next(iter(self.project.find_interpreters(self.python))) + except StopIteration: # pragma: no cover + raise VirtualenvCreateError(f"Can't find python interpreter {self.python}") + + @property + def ident(self) -> str: + """Get the identifier of this virtualenv. + self.python can be one of: + 3.8 + /usr/bin/python + 3.9.0a4 + python3.8 + """ + return self._resolved_interpreter.identifier + + def subprocess_call(self, cmd: List[str], **kwargs: Any) -> None: + self.project.core.ui.echo( + f"Run command: [green]{cmd}[/]", verbosity=termui.Verbosity.DETAIL, err=True + ) + try: + subprocess.check_call( + cmd, + stdout=subprocess.DEVNULL + if self.project.core.ui.verbosity < termui.Verbosity.DETAIL + else None, + ) + except subprocess.CalledProcessError as e: # pragma: no cover + raise VirtualenvCreateError(e) from None + + def _ensure_clean(self, location: Path, force: bool = False) -> None: + if not location.exists(): + return + if not force: + raise VirtualenvCreateError(f"The location {location} is not empty") + self.project.core.ui.echo( + f"Cleaning existing target directory {location}", err=True + ) + shutil.rmtree(location) + + def get_location(self, name: Optional[str]) -> Path: + venv_parent = Path(self.project.config["venv.location"]) + if not venv_parent.is_dir(): + venv_parent.mkdir(exist_ok=True, parents=True) + return venv_parent / f"{get_venv_prefix(self.project)}{name or self.ident}" + + def create( + self, + name: Optional[str] = None, + args: Tuple[str, ...] = (), + force: bool = False, + in_project: bool = False, + ) -> Path: + if in_project: + location = self.project.root / ".venv" + else: + location = self.get_location(name) + self._ensure_clean(location, force) + self.perform_create(location, args) + return location + + @abc.abstractmethod + def perform_create(self, location: Path, args: Tuple[str, ...]) -> None: + pass + + +class VirtualenvBackend(Backend): + def perform_create(self, location: Path, args: Tuple[str, ...]) -> None: + cmd = [ + sys.executable, + "-m", + "virtualenv", + "--no-pip", + "--no-setuptools", + "--no-wheel", + str(location), + ] + cmd.extend(["-p", str(self._resolved_interpreter.executable)]) + cmd.extend(args) + self.subprocess_call(cmd) + + +class VenvBackend(VirtualenvBackend): + def perform_create(self, location: Path, args: Tuple[str, ...]) -> None: + cmd = [ + str(self._resolved_interpreter.executable), + "-m", + "venv", + "--without-pip", + str(location), + ] + list(args) + self.subprocess_call(cmd) + + +class CondaBackend(Backend): + @property + def ident(self) -> str: + # Conda supports specifying python that doesn't exist, + # use the passed-in name directly + if self.python: + return self.python + return super().ident + + def perform_create(self, location: Path, args: Tuple[str, ...]) -> None: + if self.python: + python_ver = self.python + else: + python = self._resolved_interpreter + python_ver = f"{python.major}.{python.minor}" + if any(arg.startswith("python=") for arg in args): + raise PdmUsageError("Cannot use python= in conda creation arguments") + cmd = [ + "conda", + "create", + "--yes", + "--prefix", + str(location), + f"python={python_ver}", + *args, + ] + + self.subprocess_call(cmd) + + +BACKENDS: Mapping[str, Type[Backend]] = { + "virtualenv": VirtualenvBackend, + "venv": VenvBackend, + "conda": CondaBackend, +} diff --git a/pdm/cli/commands/venv/create.py b/pdm/cli/commands/venv/create.py new file mode 100644 index 00000000..9ceee9a3 --- /dev/null +++ b/pdm/cli/commands/venv/create.py @@ -0,0 +1,57 @@ +import argparse + +from pdm import BaseCommand, Project +from pdm.cli.commands.venv.backends import BACKENDS +from pdm.cli.options import verbose_option + + +class CreateCommand(BaseCommand): + """Create a virtualenv + + pdm venv create [-other args] + """ + + description = "Create a virtualenv" + arguments = [verbose_option] + + def add_arguments(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "-w", + "--with", + dest="backend", + choices=BACKENDS.keys(), + help="Specify the backend to create the virtualenv", + ) + parser.add_argument( + "-f", + "--force", + action="store_true", + help="Recreate if the virtualenv already exists", + ) + parser.add_argument("-n", "--name", help="Specify the name of the virtualenv") + parser.add_argument( + "python", + nargs="?", + help="Specify which python should be used to create the virtualenv", + ) + parser.add_argument( + "venv_args", + nargs=argparse.REMAINDER, + help="Additional arguments that will be passed to the backend", + ) + + def handle(self, project: Project, options: argparse.Namespace) -> None: + in_project = ( + project.config["venv.in_project"] + and not options.name + and not project.root.joinpath(".venv").exists() + ) + backend: str = options.backend or project.config["venv.backend"] + venv_backend = BACKENDS[backend](project, options.python) + with project.core.ui.open_spinner( + f"Creating virtualenv using [green]{backend}[/]..." + ): + path = venv_backend.create( + options.name, options.venv_args, options.force, in_project + ) + project.core.ui.echo(f"Virtualenv [green]{path}[/] is created successfully") diff --git a/pdm/cli/commands/venv/list.py b/pdm/cli/commands/venv/list.py new file mode 100644 index 00000000..d6dedf70 --- /dev/null +++ b/pdm/cli/commands/venv/list.py @@ -0,0 +1,23 @@ +import argparse +from pathlib import Path + +from pdm.cli.commands.base import BaseCommand +from pdm.cli.commands.venv.utils import iter_venvs +from pdm.cli.options import verbose_option +from pdm.project import Project + + +class ListCommand(BaseCommand): + """List all virtualenvs associated with this project""" + + arguments = [verbose_option] + + def handle(self, project: Project, options: argparse.Namespace) -> None: + project.core.ui.echo("Virtualenvs created with this project:\n") + for ident, venv in iter_venvs(project): + saved_python = project.project_config.get("python.path") + if saved_python and Path(saved_python).parent.parent == venv: + mark = "*" + else: + mark = "-" + project.core.ui.echo(f"{mark} [green]{ident}[/]: {venv}") diff --git a/pdm/cli/commands/venv/purge.py b/pdm/cli/commands/venv/purge.py new file mode 100644 index 00000000..28398b50 --- /dev/null +++ b/pdm/cli/commands/venv/purge.py @@ -0,0 +1,65 @@ +import argparse +import shutil + +from pdm import Project, termui +from pdm.cli.commands.base import BaseCommand +from pdm.cli.commands.venv.utils import iter_central_venvs +from pdm.cli.options import verbose_option + + +class PurgeCommand(BaseCommand): + """Purge selected/all created Virtualenvs""" + + arguments = [verbose_option] + + def add_arguments(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "-f", + "--force", + action="store_true", + help="Force purging without prompting for confirmation", + ) + parser.add_argument( + "-i", + "--interactive", + action="store_true", + help="Interatively purge selected Virtualenvs", + ) + + def handle(self, project: Project, options: argparse.Namespace) -> None: + + all_central_venvs = list(iter_central_venvs(project)) + if not all_central_venvs: + project.core.ui.echo("No virtualenvs to purge, quitting.", style="green") + return + + if not options.force: + project.core.ui.echo( + "The following Virtualenvs will be purged:", style="red" + ) + for i, venv in enumerate(all_central_venvs): + project.core.ui.echo(f"{i}. [green]{venv[0]}[/]") + + if not options.interactive: + if options.force or termui.confirm("continue?", default=True): + return self.del_all_venvs(project) + + selection = termui.ask( + "Please select", + choices=([str(i) for i in range(len(all_central_venvs))] + ["all", "none"]), + default="none", + show_choices=False, + ) + + if selection == "all": + self.del_all_venvs(project) + elif selection != "none": + for i, venv in enumerate(all_central_venvs): + if i == int(selection): + shutil.rmtree(venv[1]) + project.core.ui.echo("Purged successfully!") + + def del_all_venvs(self, project: Project) -> None: + for _, venv in iter_central_venvs(project): + shutil.rmtree(venv) + project.core.ui.echo("Purged successfully!") diff --git a/pdm/cli/commands/venv/remove.py b/pdm/cli/commands/venv/remove.py new file mode 100644 index 00000000..9ce3c280 --- /dev/null +++ b/pdm/cli/commands/venv/remove.py @@ -0,0 +1,47 @@ +import argparse +import shutil +from pathlib import Path + +from pdm import Project, termui +from pdm.cli.commands.base import BaseCommand +from pdm.cli.commands.venv.utils import iter_venvs +from pdm.cli.options import verbose_option + + +class RemoveCommand(BaseCommand): + """Remove the virtualenv with the given name""" + + arguments = [verbose_option] + + def add_arguments(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "-y", + "--yes", + action="store_true", + help="Answer yes on the following question", + ) + parser.add_argument("env", help="The key of the virtualenv") + + def handle(self, project: Project, options: argparse.Namespace) -> None: + project.core.ui.echo("Virtualenvs created with this project:") + for ident, venv in iter_venvs(project): + if ident == options.env: + if options.yes or termui.confirm( + f"[yellow]Will remove: [green]{venv}[/], continue?", default=True + ): + shutil.rmtree(venv) + if ( + project.project_config.get("python.path") + and Path(project.project_config["python.path"]).parent.parent + == venv + ): + del project.project_config["python.path"] + project.core.ui.echo("Removed successfully!") + break + else: + project.core.ui.echo( + f"No virtualenv with key [green]{options.env}[/] is found", + style="yellow", + err=True, + ) + raise SystemExit(1) diff --git a/pdm/cli/commands/venv/utils.py b/pdm/cli/commands/venv/utils.py new file mode 100644 index 00000000..341fd0e8 --- /dev/null +++ b/pdm/cli/commands/venv/utils.py @@ -0,0 +1,80 @@ +import base64 +import hashlib +import sys +from pathlib import Path +from typing import Iterable, Optional, Tuple, Type, TypeVar +from findpython import PythonVersion + +from findpython.providers import BaseProvider + +from pdm.project import Project + +IS_WIN = sys.platform == "win32" +BIN_DIR = "Scripts" if IS_WIN else "bin" + + +def hash_path(path: str) -> str: + """Generate a hash for the given path.""" + return base64.urlsafe_b64encode(hashlib.md5(path.encode()).digest()).decode()[:8] + + +def get_in_project_venv_python(root: Path) -> Optional[Path]: + """Get the python interpreter path of venv-in-project""" + for possible_dir in (".venv", "venv", "env"): + venv_python = get_venv_python(root / possible_dir) + if venv_python.exists(): + return venv_python + return None + + +def get_venv_prefix(project: Project) -> str: + """Get the venv prefix for the project""" + path = project.root + name_hash = hash_path(path.as_posix()) + return f"{path.name}-{name_hash}-" + + +def iter_venvs(project: Project) -> Iterable[Tuple[str, Path]]: + """Return an iterable of venv paths associated with the project""" + in_project_venv_python = get_in_project_venv_python(project.root) + if in_project_venv_python is not None: + yield "in-project", Path(in_project_venv_python).parent.parent + venv_prefix = get_venv_prefix(project) + venv_parent = Path(project.config["venv.location"]) + for venv in venv_parent.glob(f"{venv_prefix}*"): + ident = venv.name[len(venv_prefix) :] + yield ident, venv + + +def get_venv_python(venv: Path) -> Path: + """Get the interpreter path inside the given venv.""" + suffix = ".exe" if IS_WIN else "" + return venv / BIN_DIR / f"python{suffix}" + + +def iter_central_venvs(project: Project) -> Iterable[Tuple[str, Path]]: + """Return an iterable of all managed venvs and their paths.""" + venv_parent = Path(project.config["venv.location"]) + for venv in venv_parent.glob("*"): + ident = venv.name + yield ident, venv + + +T = TypeVar("T", bound=BaseProvider) + + +class VenvProvider(BaseProvider): + """A Python provider for project venv pythons""" + + def __init__(self, project: Project) -> None: + self.project = project + + @classmethod + def create(cls: Type[T]) -> Optional[T]: # pragma: no cover + return None + + def find_pythons(self) -> Iterable[PythonVersion]: + for _, venv in iter_venvs(self.project): + python = get_venv_python(venv) + if python.exists(): + yield PythonVersion(python, _interpreter=python, keep_symlink=True) diff --git a/pdm/cli/completions/pdm.bash b/pdm/cli/completions/pdm.bash index 302d63ba..fe88f1d4 100644 --- a/pdm/cli/completions/pdm.bash +++ b/pdm/cli/completions/pdm.bash @@ -1,7 +1,7 @@ # BASH completion script for pdm # Generated by pycomplete 0.3.2 -_pdm_25182a7ef85b840e_complete() +_pdm_a919b69078acdf0a_complete() { local cur script coms opts com COMPREPLY=() @@ -101,7 +101,7 @@ _pdm_25182a7ef85b840e_complete() ;; (sync) - opts="--clean --dev --dry-run --global --group --help --lockfile --only-keep --no-default --no-editable --no-isolation --no-self --production --project --reinstall --skip --verbose" + opts="--clean --dev --dry-run --global --group --help --lockfile --no-default --no-editable --no-isolation --no-self --only-keep --production --project --reinstall --skip --verbose" ;; (update) @@ -109,7 +109,11 @@ _pdm_25182a7ef85b840e_complete() ;; (use) - opts="--first --global --help --ignore-remembered --project --verbose" + opts="--first --global --help --ignore-remembered --project --skip --verbose" + ;; + + (venv) + opts="--help" ;; esac @@ -122,7 +126,7 @@ _pdm_25182a7ef85b840e_complete() # completing for a command if [[ $cur == $com ]]; then - coms="add build cache completion config export import info init install list lock plugin publish remove run search show sync update use" + coms="add build cache completion config export import info init install list lock plugin publish remove run search show sync update use venv" COMPREPLY=($(compgen -W "${coms}" -- ${cur})) __ltrim_colon_completions "$cur" @@ -131,4 +135,4 @@ _pdm_25182a7ef85b840e_complete() fi } -complete -o default -F _pdm_25182a7ef85b840e_complete pdm +complete -o default -F _pdm_a919b69078acdf0a_complete pdm diff --git a/pdm/cli/completions/pdm.fish b/pdm/cli/completions/pdm.fish index dd837905..e1a32f18 100644 --- a/pdm/cli/completions/pdm.fish +++ b/pdm/cli/completions/pdm.fish @@ -1,9 +1,9 @@ # FISH completion script for pdm # Generated by pycomplete 0.3.2 -function __fish_pdm_7426f3abf02b4bb8_complete_no_subcommand +function __fish_pdm_a919b69078acdf0a_complete_no_subcommand for i in (commandline -opc) - if contains -- $i add build cache completion config export import info init install list lock plugin publish remove run search show sync update use + if contains -- $i add build cache completion config export import info init install list lock plugin publish remove run search show sync update use venv return 1 end end @@ -11,35 +11,36 @@ function __fish_pdm_7426f3abf02b4bb8_complete_no_subcommand end # global options -complete -c pdm -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -l config -d 'Specify another config file path(env var: PDM_CONFIG_FILE)' -complete -c pdm -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -l help -d 'show this help message and exit' -complete -c pdm -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -l ignore-python -d 'Ignore the Python path saved in the pdm.toml config' -complete -c pdm -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -l pep582 -d 'Print the command line to be eval\'d by the shell' -complete -c pdm -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -l verbose -d '-v for detailed output and -vv for more detailed' -complete -c pdm -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -l version -d 'Show version' +complete -c pdm -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -l config -d 'Specify another config file path(env var: PDM_CONFIG_FILE)' +complete -c pdm -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -l help -d 'show this help message and exit' +complete -c pdm -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -l ignore-python -d 'Ignore the Python path saved in the pdm.toml config' +complete -c pdm -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -l pep582 -d 'Print the command line to be eval\'d by the shell' +complete -c pdm -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -l verbose -d '-v for detailed output and -vv for more detailed' +complete -c pdm -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -l version -d 'Show version' # commands -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a add -d 'Add package(s) to pyproject.toml and install them' -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a build -d 'Build artifacts for distribution' -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a cache -d 'Control the caches of PDM' -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a completion -d 'Generate completion scripts for the given shell' -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a config -d 'Display the current configuration' -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a export -d 'Export the locked packages set to other formats' -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a import -d 'Import project metadata from other formats' -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a info -d 'Show the project information' -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a init -d 'Initialize a pyproject.toml for PDM' -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a install -d 'Install dependencies from lock file' -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a list -d 'List packages installed in the current working set' -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a lock -d 'Resolve and lock dependencies' -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a plugin -d 'Manage the PDM plugins' -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a publish -d 'Build and publish the project to PyPI' -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a remove -d 'Remove packages from pyproject.toml' -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a run -d 'Run commands or scripts with local packages loaded' -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a search -d 'Search for PyPI packages' -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a show -d 'Show the package information' -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a sync -d 'Synchronize the current working set with lock file' -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a update -d 'Update package(s) in pyproject.toml' -complete -c pdm -f -n '__fish_pdm_7426f3abf02b4bb8_complete_no_subcommand' -a use -d 'Use the given python version or path as base interpreter' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a add -d 'Add package(s) to pyproject.toml and install them' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a build -d 'Build artifacts for distribution' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a cache -d 'Control the caches of PDM' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a completion -d 'Generate completion scripts for the given shell' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a config -d 'Display the current configuration' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a export -d 'Export the locked packages set to other formats' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a import -d 'Import project metadata from other formats' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a info -d 'Show the project information' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a init -d 'Initialize a pyproject.toml for PDM' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a install -d 'Install dependencies from lock file' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a list -d 'List packages installed in the current working set' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a lock -d 'Resolve and lock dependencies' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a plugin -d 'Manage the PDM plugins' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a publish -d 'Build and publish the project to PyPI' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a remove -d 'Remove packages from pyproject.toml' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a run -d 'Run commands or scripts with local packages loaded' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a search -d 'Search for PyPI packages' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a show -d 'Show the package information' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a sync -d 'Synchronize the current working set with lock file' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a update -d 'Update package(s) in pyproject.toml' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a use -d 'Use the given python version or path as base interpreter' +complete -c pdm -f -n '__fish_pdm_a919b69078acdf0a_complete_no_subcommand' -a venv -d 'Virtualenv management' # command options @@ -233,18 +234,18 @@ complete -c pdm -A -n '__fish_seen_subcommand_from show' -l verbose -d '-v for d complete -c pdm -A -n '__fish_seen_subcommand_from show' -l version -d 'Show version' # sync -complete -c pdm -A -n '__fish_seen_subcommand_from sync' -l clean -d 'clean unused packages' +complete -c pdm -A -n '__fish_seen_subcommand_from sync' -l clean -d 'clean packages not in the lockfile' complete -c pdm -A -n '__fish_seen_subcommand_from sync' -l dev -d 'Select dev dependencies' complete -c pdm -A -n '__fish_seen_subcommand_from sync' -l dry-run -d 'Show the difference only and don\'t perform any action' complete -c pdm -A -n '__fish_seen_subcommand_from sync' -l global -d 'Use the global project, supply the project root with `-p` option' complete -c pdm -A -n '__fish_seen_subcommand_from sync' -l group -d 'Select group of optional-dependencies or dev-dependencies(with -d). Can be supplied multiple times, use ":all" to include all groups under the same species.' complete -c pdm -A -n '__fish_seen_subcommand_from sync' -l help -d 'show this help message and exit' complete -c pdm -A -n '__fish_seen_subcommand_from sync' -l lockfile -d 'Specify another lockfile path. Default: pdm.lock. [env var: PDM_LOCKFILE]' -complete -c pdm -A -n '__fish_seen_subcommand_from sync' -l only-keep -d 'don\'t clean unused packages' complete -c pdm -A -n '__fish_seen_subcommand_from sync' -l no-default -d 'Don\'t include dependencies from the default group' complete -c pdm -A -n '__fish_seen_subcommand_from sync' -l no-editable -d 'Install non-editable versions for all packages' complete -c pdm -A -n '__fish_seen_subcommand_from sync' -l no-isolation -d 'Do not isolate the build in a clean environment' complete -c pdm -A -n '__fish_seen_subcommand_from sync' -l no-self -d 'Don\'t install the project itself' +complete -c pdm -A -n '__fish_seen_subcommand_from sync' -l only-keep -d 'only keep the selected packages' complete -c pdm -A -n '__fish_seen_subcommand_from sync' -l production -d 'Unselect dev dependencies' complete -c pdm -A -n '__fish_seen_subcommand_from sync' -l project -d 'Specify another path as the project root, which changes the base of pyproject.toml and __pypackages__' complete -c pdm -A -n '__fish_seen_subcommand_from sync' -l reinstall -d 'Force reinstall existing dependencies' @@ -284,4 +285,8 @@ complete -c pdm -A -n '__fish_seen_subcommand_from use' -l global -d 'Use the gl complete -c pdm -A -n '__fish_seen_subcommand_from use' -l help -d 'show this help message and exit' complete -c pdm -A -n '__fish_seen_subcommand_from use' -l ignore-remembered -d 'Ignore the remembered selection' complete -c pdm -A -n '__fish_seen_subcommand_from use' -l project -d 'Specify another path as the project root, which changes the base of pyproject.toml and __pypackages__' +complete -c pdm -A -n '__fish_seen_subcommand_from use' -l skip -d 'Skip some tasks and/or hooks by their comma-separated names. Can be supplied multiple times. Use ":all" to skip all hooks. Use ":pre" and ":post" to skip all pre or post hooks.' complete -c pdm -A -n '__fish_seen_subcommand_from use' -l verbose -d '-v for detailed output and -vv for more detailed' + +# venv +complete -c pdm -A -n '__fish_seen_subcommand_from venv' -l help -d 'show this help message and exit' diff --git a/pdm/cli/completions/pdm.ps1 b/pdm/cli/completions/pdm.ps1 index 0964087e..465de214 100644 --- a/pdm/cli/completions/pdm.ps1 +++ b/pdm/cli/completions/pdm.ps1 @@ -380,7 +380,7 @@ function TabExpansion($line, $lastWord) { "use" { $completer.AddOpts( @( - [Option]::new(@("--global", "-g", "-f", "--first", "-i", "--ignore-remembered")), + [Option]::new(@("--global", "-g", "-f", "--first", "-i", "--ignore-remembered", "--skip")), $projectOption )) break diff --git a/pdm/models/environment.py b/pdm/models/environment.py index 8a8be0c8..be0435ba 100644 --- a/pdm/models/environment.py +++ b/pdm/models/environment.py @@ -24,7 +24,7 @@ from pdm.models.in_process import ( from pdm.models.python import PythonInfo from pdm.models.session import PDMSession from pdm.models.working_set import WorkingSet -from pdm.utils import cached_property, get_index_urls, is_venv_python, pdm_scheme +from pdm.utils import cached_property, get_index_urls, get_venv_like_prefix, pdm_scheme if TYPE_CHECKING: from pdm._types import Source @@ -241,7 +241,7 @@ class GlobalEnvironment(Environment): is_global = True def get_paths(self) -> dict[str, str]: - is_venv = is_venv_python(self.interpreter.executable) + is_venv = bool(get_venv_like_prefix(self.interpreter.executable)) paths = get_sys_config_paths( str(self.interpreter.executable), kind="user" diff --git a/pdm/project/config.py b/pdm/project/config.py index 1aaf0cb5..5634d43a 100644 --- a/pdm/project/config.py +++ b/pdm/project/config.py @@ -186,7 +186,7 @@ class Config(MutableMapping[str, str]): ), "python.use_venv": ConfigItem( "Install packages into the activated venv site packages instead of PEP 582", - False, + True, env_var="PDM_USE_VENV", coerce=ensure_boolean, replace="use_venv", @@ -205,6 +205,22 @@ class Config(MutableMapping[str, str]): env_var="PDM_PYPI_JSON_API", coerce=ensure_boolean, ), + "venv.location": ConfigItem( + "Parent directory for virtualenvs", + os.path.join(platformdirs.user_data_dir("pdm"), "venvs"), + global_only=True, + ), + "venv.backend": ConfigItem( + "Default backend to create virtualenv", + default="virtualenv", + env_var="PDM_VENV_BACKEND", + ), + "venv.in_project": ConfigItem( + "Create virtualenv in `.venv` under project root", + default=True, + env_var="PDM_VENV_IN_PROJECT", + coerce=ensure_boolean, + ), } @classmethod diff --git a/pdm/project/core.py b/pdm/project/core.py index 35917ae2..f91d15fa 100644 --- a/pdm/project/core.py +++ b/pdm/project/core.py @@ -34,7 +34,6 @@ from pdm.utils import ( expand_env_vars_in_auth, find_project_root, find_python_in_path, - get_in_project_venv_python, get_venv_like_prefix, normalize_name, path_to_url, @@ -239,18 +238,69 @@ class Project: def get_environment(self) -> Environment: """Get the environment selected by this project""" + from pdm.cli.commands.venv.utils import get_venv_python, iter_venvs + if self.is_global: env = GlobalEnvironment(self) # Rewrite global project's python requires to be # compatible with the exact version env.python_requires = PySpecSet(f"=={self.python.version}") return env - if self.config["python.use_venv"] and get_venv_like_prefix( - self.python.executable + if not self.config["python.use_venv"]: + return Environment(self) + if self.project_config.get("python.path") and not os.getenv( + "PDM_IGNORE_SAVED_PYTHON" ): - # Only recognize venv created by python -m venv and virtualenv>20 + return ( + GlobalEnvironment(self) + if get_venv_like_prefix(self.python.executable) is not None + else Environment(self) + ) + if os.getenv("VIRTUAL_ENV"): + venv = cast(str, os.getenv("VIRTUAL_ENV")) + self.core.ui.echo( + f"Detected inside an active virtualenv [green]{venv}[/], reuse it.", + style="yellow", + err=True, + ) + # Temporary usage, do not save in .pdm.toml + self._python = PythonInfo.from_path(get_venv_python(Path(venv))) return GlobalEnvironment(self) - return Environment(self) + existing_venv = next((venv for _, venv in iter_venvs(self)), None) + if existing_venv: + self.core.ui.echo( + f"Virtualenv [green]{existing_venv}[/] is reused.", + err=True, + ) + path = existing_venv + elif self.root.joinpath("__pypackages__").exists(): + self.core.ui.echo( + "__pypackages__ is detected, use the PEP 582 mode", + style="green", + err=True, + ) + return Environment(self) + else: + # Create a virtualenv using the selected Python interpreter + self.core.ui.echo( + "python.use_venv is on, creating a virtualenv for this project...", + style="yellow", + err=True, + ) + path = self._create_virtualenv() + self.python = PythonInfo.from_path(get_venv_python(path)) + return GlobalEnvironment(self) + + def _create_virtualenv(self) -> Path: + from pdm.cli.commands.venv.backends import BACKENDS + + backend: str = self.config["venv.backend"] + venv_backend = BACKENDS[backend](self, None) + path = venv_backend.create(in_project=self.config["venv.in_project"]) + self.core.ui.echo( + f"Virtualenv is created successfully at [green]{path}[/]", err=True + ) + return path @property def environment(self) -> Environment: @@ -630,13 +680,6 @@ dependencies = ["pip", "setuptools", "wheel"] yield PythonInfo.from_path(pyenv_shim) elif os.path.exists(pyenv_shim.replace("python3", "python")): yield PythonInfo.from_path(pyenv_shim.replace("python3", "python")) - if config.get("python.use_venv"): - python = get_in_project_venv_python(self.root) - if python: - yield PythonInfo.from_path(python) - python = shutil.which("python") - if python: - yield PythonInfo.from_path(python) args = [] else: if not all(c.isdigit() for c in python_spec.split(".")): @@ -650,9 +693,21 @@ dependencies = ["pip", "setuptools", "wheel"] yield PythonInfo.from_path(python) return args = [int(v) for v in python_spec.split(".") if v != ""] - finder = Finder(resolve_symlinks=True) + finder = self._get_python_finder() for entry in finder.find_all(*args): yield PythonInfo(entry) if not python_spec: + python = shutil.which("python") + if python: + yield PythonInfo.from_path(python) + # Return the host Python as well this_python = getattr(sys, "_base_executable", sys.executable) yield PythonInfo.from_path(this_python) + + def _get_python_finder(self) -> Finder: + from pdm.cli.commands.venv.utils import VenvProvider + + finder = Finder(resolve_symlinks=True) + if self.config["python.use_venv"]: + finder._providers.insert(0, VenvProvider(self)) + return finder diff --git a/pdm/termui.py b/pdm/termui.py index 57de9d52..77532235 100644 --- a/pdm/termui.py +++ b/pdm/termui.py @@ -69,6 +69,7 @@ def style( def confirm(*args: str, **kwargs: Any) -> str: + kwargs.setdefault("default", False) return Confirm.ask(*args, **kwargs) diff --git a/pdm/utils.py b/pdm/utils.py index 31f00ea6..94a92303 100644 --- a/pdm/utils.py +++ b/pdm/utils.py @@ -187,21 +187,6 @@ def add_ssh_scheme_to_git_uri(uri: str) -> str: return uri -def get_in_project_venv_python(root: Path) -> Path | None: - """Get the python interpreter path of venv-in-project""" - if os.name == "nt": - suffix = ".exe" - scripts = "Scripts" - else: - suffix = "" - scripts = "bin" - for possible_dir in ("venv", ".venv", "env"): - if (root / possible_dir / scripts / f"python{suffix}").exists(): - venv = root / possible_dir - return venv / scripts / f"python{suffix}" - return None - - @contextlib.contextmanager def atomic_open_for_write( filename: str | Path, *, mode: str = "w", encoding: str = "utf-8" @@ -345,16 +330,6 @@ def is_path_relative_to(path: str | Path, other: str | Path) -> bool: return True -def is_venv_python(interpreter: str | Path) -> bool: - """Check if the given interpreter path is from a virtualenv""" - interpreter = Path(interpreter) - if interpreter.parent.parent.joinpath("pyvenv.cfg").exists(): - return True - - virtual_env = os.getenv("VIRTUAL_ENV") - return bool(virtual_env and is_path_relative_to(interpreter, virtual_env)) - - def get_venv_like_prefix(interpreter: str | Path) -> Path | None: """Check if the given interpreter path is from a virtualenv, and return the prefix if found. diff --git a/pyproject.toml b/pyproject.toml index 7ca66843..23d943b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,24 +9,25 @@ requires-python = ">=3.7" license = {text = "MIT"} dependencies = [ "blinker", - "findpython", - "importlib-metadata; python_version < \"3.8\"", - "installer>=0.5.1,<0.6", - "packaging", - "pdm-pep517>=1.0.0,<2.0.0", - "pep517>=0.11.0", "pip>=20", + "packaging", "platformdirs", - "python-dotenv>=0.15", - "resolvelib>=0.8,<0.9", "rich>=12.3.0", - "shellingham>=1.3.2", - "tomli>=1.1.0; python_version < \"3.11\"", + "virtualenv>=20", + "pep517>=0.11.0", + "requests-toolbelt", + "findpython>=0.2.0", "tomlkit>=0.8.0,<1", - "typing-extensions; python_version < \"3.8\"", + "shellingham>=1.3.2", + "python-dotenv>=0.15", + "resolvelib>=0.8,<0.9", "unearth>=0.4.1,<0.5.0", + "installer>=0.5.1,<0.6", + "pdm-pep517>=1.0.0,<2.0.0", "cachecontrol[filecache]>=0.12.11", - "requests-toolbelt", + "tomli>=1.1.0; python_version < \"3.11\"", + "typing-extensions; python_version < \"3.8\"", + "importlib-metadata; python_version < \"3.8\"", ] name = "pdm" description = "A modern Python package and dependency manager supporting the latest PEP standards" diff --git a/tests/cli/test_build.py b/tests/cli/test_build.py index d39da64e..b4278bcb 100644 --- a/tests/cli/test_build.py +++ b/tests/cli/test_build.py @@ -1,4 +1,3 @@ -import os import tarfile import zipfile @@ -169,7 +168,7 @@ def test_build_with_no_isolation(fixture_project, invoke, isolated): assert result.exit_code == int(isolated) -def test_build_ignoring_pip_environment(fixture_project): +def test_build_ignoring_pip_environment(fixture_project, monkeypatch): project = fixture_project("demo-module") - os.environ["PIP_REQUIRE_VIRTUALENV"] = "1" + monkeypatch.setenv("PIP_REQUIRE_VIRTUALENV", "1") actions.do_build(project) diff --git a/tests/cli/test_config.py b/tests/cli/test_config.py index f14edd07..08b5fd94 100644 --- a/tests/cli/test_config.py +++ b/tests/cli/test_config.py @@ -1,5 +1,3 @@ -import os - import pytest from pdm.exceptions import PdmUsageError @@ -53,8 +51,8 @@ def test_config_del_command(project, invoke): assert result.output.strip() == "True" -def test_config_env_var_shadowing(project, invoke): - os.environ["PDM_PYPI_URL"] = "https://example.org/simple" +def test_config_env_var_shadowing(project, invoke, monkeypatch): + monkeypatch.setenv("PDM_PYPI_URL", "https://example.org/simple") result = invoke(["config", "pypi.url"], obj=project) assert result.output.strip() == "https://example.org/simple" @@ -63,7 +61,7 @@ def test_config_env_var_shadowing(project, invoke): result = invoke(["config", "pypi.url"], obj=project) assert result.output.strip() == "https://example.org/simple" - del os.environ["PDM_PYPI_URL"] + monkeypatch.delenv("PDM_PYPI_URL") result = invoke(["config", "pypi.url"], obj=project) assert result.output.strip() == "https://test.pypi.org/pypi" @@ -102,14 +100,14 @@ def test_rename_deprecated_config(tmp_path, invoke): ) -def test_specify_config_file(tmp_path, invoke): +def test_specify_config_file(tmp_path, invoke, monkeypatch): tmp_path.joinpath("global_config.toml").write_text("project_max_depth = 9\n") with cd(tmp_path): result = invoke(["-c", "global_config.toml", "config", "project_max_depth"]) assert result.exit_code == 0 assert result.output.strip() == "9" - os.environ["PDM_CONFIG_FILE"] = "global_config.toml" + monkeypatch.setenv("PDM_CONFIG_FILE", "global_config.toml") result = invoke(["config", "project_max_depth"]) assert result.exit_code == 0 assert result.output.strip() == "9" diff --git a/tests/cli/test_hooks.py b/tests/cli/test_hooks.py index bbfa9953..4da7491a 100644 --- a/tests/cli/test_hooks.py +++ b/tests/cli/test_hooks.py @@ -1,4 +1,5 @@ import shlex +import sys from collections import namedtuple from textwrap import dedent @@ -9,6 +10,8 @@ from pdm.cli.hooks import KNOWN_HOOKS from pdm.cli.options import from_splitted_env from pdm.models.requirements import parse_requirement +pytestmark = pytest.mark.usefixtures("repository", "working_set", "local_finder") + def test_pre_script_fail_fast(project, invoke, capfd, mocker): project.tool_settings["scripts"] = { @@ -193,6 +196,7 @@ def test_skip_option_default_from_env(env, expected, monkeypatch): HookSpecs = namedtuple("HookSpecs", ["command", "hooks", "fixtures"]) +this_python_version = f"{sys.version_info[0]}.{sys.version_info[1]}" KNOWN_COMMAND_HOOKS = ( ("add", "add requests", ("pre_lock", "post_lock"), ["working_set"]), @@ -214,7 +218,7 @@ KNOWN_COMMAND_HOOKS = ( ("remove", "remove requests", ("pre_lock", "post_lock"), ["lock"]), ("sync", "sync", ("pre_install", "post_install"), ["lock"]), ("update", "update", ("pre_install", "post_install", "pre_lock", "post_lock"), []), - ("use", "use -f 3.7", ("post_use",), []), + ("use", f"use -f {this_python_version}", ("post_use",), []), ) parametrize_with_commands = pytest.mark.parametrize( diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py new file mode 100644 index 00000000..da8723b0 --- /dev/null +++ b/tests/cli/test_init.py @@ -0,0 +1,111 @@ +from unittest.mock import ANY + +import pytest + +from pdm.cli import actions + + +def test_init_validate_python_requires(project_no_init): + with pytest.raises(ValueError): + actions.do_init(project_no_init, python_requires="3.7") + + +def test_init_command(project_no_init, invoke, mocker): + mocker.patch( + "pdm.cli.commands.init.get_user_email_from_git", + return_value=("Testing", "me@example.org"), + ) + do_init = mocker.patch.object(actions, "do_init") + invoke(["init"], input="\n\n\n\n\n\n", strict=True, obj=project_no_init) + python_version = f"{project_no_init.python.major}.{project_no_init.python.minor}" + do_init.assert_called_with( + project_no_init, + name="", + version="", + description="", + license="MIT", + author="Testing", + email="me@example.org", + python_requires=f">={python_version}", + hooks=ANY, + ) + + +def test_init_command_library(project_no_init, invoke, mocker): + mocker.patch( + "pdm.cli.commands.init.get_user_email_from_git", + return_value=("Testing", "me@example.org"), + ) + do_init = mocker.patch.object(actions, "do_init") + result = invoke( + ["init"], + input="\ny\ntest-project\n\nTest Project\n\n\n\n\n", + obj=project_no_init, + ) + assert result.exit_code == 0 + python_version = f"{project_no_init.python.major}.{project_no_init.python.minor}" + do_init.assert_called_with( + project_no_init, + name="test-project", + version="0.1.0", + description="Test Project", + license="MIT", + author="Testing", + email="me@example.org", + python_requires=f">={python_version}", + hooks=ANY, + ) + + +def test_init_non_interactive(project_no_init, invoke, mocker): + mocker.patch( + "pdm.cli.commands.init.get_user_email_from_git", + return_value=("Testing", "me@example.org"), + ) + do_init = mocker.patch.object(actions, "do_init") + do_use = mocker.patch.object(actions, "do_use") + result = invoke(["init", "-n"], obj=project_no_init) + assert result.exit_code == 0 + python_version = f"{project_no_init.python.major}.{project_no_init.python.minor}" + do_use.assert_called_once_with( + project_no_init, + ANY, + True, + ignore_requires_python=True, + hooks=ANY, + ) + do_init.assert_called_with( + project_no_init, + name="", + version="", + description="", + license="MIT", + author="Testing", + email="me@example.org", + python_requires=f">={python_version}", + hooks=ANY, + ) + + +def test_init_auto_create_venv(project_no_init, invoke, mocker): + mocker.patch("pdm.cli.commands.init.get_venv_like_prefix", return_value=None) + project_no_init.project_config["python.use_venv"] = True + result = invoke(["init"], input="\n\n\n\n\n\n\n", obj=project_no_init) + assert result.exit_code == 0 + assert ( + project_no_init.python.executable.parent.parent + == project_no_init.root / ".venv" + ) + + +def test_init_auto_create_venv_answer_no(project_no_init, invoke, mocker): + mocker.patch("pdm.cli.commands.init.get_venv_like_prefix", return_value=None) + creator = mocker.patch("pdm.cli.commands.venv.backends.Backend.create") + project_no_init.project_config["python.use_venv"] = True + result = invoke(["init"], input="\nn\n\n\n\n\n\n\n", obj=project_no_init) + assert result.exit_code == 0 + creator.assert_not_called() + assert ( + project_no_init.python.executable.parent.parent + != project_no_init.root / ".venv" + ) diff --git a/tests/cli/test_others.py b/tests/cli/test_others.py index 142a4da5..662ea90b 100644 --- a/tests/cli/test_others.py +++ b/tests/cli/test_others.py @@ -1,5 +1,3 @@ -from unittest.mock import ANY - import pytest from pdm.cli import actions @@ -33,11 +31,6 @@ def test_project_no_init_error(project_no_init): handler(project_no_init) -def test_init_validate_python_requires(project_no_init): - with pytest.raises(ValueError): - actions.do_init(project_no_init, python_requires="3.7") - - def test_help_option(invoke): result = invoke(["--help"]) assert "Usage: pdm [-h]" in result.output @@ -78,82 +71,6 @@ def test_uncaught_error(invoke, mocker): assert isinstance(result.exception, RuntimeError) -def test_init_command(project_no_init, invoke, mocker): - mocker.patch( - "pdm.cli.commands.init.get_user_email_from_git", - return_value=("Testing", "me@example.org"), - ) - do_init = mocker.patch.object(actions, "do_init") - invoke(["init"], input="\n\n\n\n\n\n", strict=True, obj=project_no_init) - python_version = f"{project_no_init.python.major}.{project_no_init.python.minor}" - do_init.assert_called_with( - project_no_init, - name="", - version="", - description="", - license="MIT", - author="Testing", - email="me@example.org", - python_requires=f">={python_version}", - hooks=ANY, - ) - - -def test_init_command_library(project_no_init, invoke, mocker): - mocker.patch( - "pdm.cli.commands.init.get_user_email_from_git", - return_value=("Testing", "me@example.org"), - ) - do_init = mocker.patch.object(actions, "do_init") - result = invoke( - ["init"], - input="\ny\ntest-project\n\nTest Project\n\n\n\n\n", - obj=project_no_init, - ) - assert result.exit_code == 0 - python_version = f"{project_no_init.python.major}.{project_no_init.python.minor}" - do_init.assert_called_with( - project_no_init, - name="test-project", - version="0.1.0", - description="Test Project", - license="MIT", - author="Testing", - email="me@example.org", - python_requires=f">={python_version}", - hooks=ANY, - ) - - -def test_init_non_interactive(project_no_init, invoke, mocker): - mocker.patch( - "pdm.cli.commands.init.get_user_email_from_git", - return_value=("Testing", "me@example.org"), - ) - do_init = mocker.patch.object(actions, "do_init") - do_use = mocker.patch.object(actions, "do_use") - result = invoke(["init", "-n"], obj=project_no_init) - assert result.exit_code == 0 - python_version = f"{project_no_init.python.major}.{project_no_init.python.minor}" - do_use.assert_called_once_with( - project_no_init, - ANY, - True, - hooks=ANY, - ) - do_init.assert_called_with( - project_no_init, - name="", - version="", - description="", - license="MIT", - author="Testing", - email="me@example.org", - python_requires=f">={python_version}", - hooks=ANY, - ) - - @pytest.mark.parametrize( "filename", [ diff --git a/tests/cli/test_run.py b/tests/cli/test_run.py index 8a5ce41e..8a404701 100644 --- a/tests/cli/test_run.py +++ b/tests/cli/test_run.py @@ -189,7 +189,7 @@ def test_run_script_with_extra_args(project, invoke, capfd): assert out.splitlines()[-3:] == ["-a", "-b", "-c"] -def test_run_expand_env_vars(project, invoke, capfd): +def test_run_expand_env_vars(project, invoke, capfd, monkeypatch): (project.root / "test_script.py").write_text("import os; print(os.getenv('FOO'))") project.tool_settings["scripts"] = { "test_cmd": 'python -c "foo, bar = 0, 1;print($FOO)"', @@ -201,7 +201,7 @@ def test_run_expand_env_vars(project, invoke, capfd): project.write_pyproject() capfd.readouterr() with cd(project.root): - os.environ["FOO"] = "bar" + monkeypatch.setenv("FOO", "bar") invoke(["run", "test_cmd"], obj=project) assert capfd.readouterr()[0].strip() == "1" diff --git a/tests/cli/test_venv.py b/tests/cli/test_venv.py new file mode 100644 index 00000000..2db2a1ce --- /dev/null +++ b/tests/cli/test_venv.py @@ -0,0 +1,246 @@ +import os +import re +import shutil +import sys +from unittest.mock import ANY + +import pytest + +from pdm.cli.commands.venv import backends +from pdm.cli.commands.venv.utils import get_venv_prefix + + +@pytest.fixture() +def fake_create(monkeypatch): + def fake_create(self, location, *args): + location.mkdir(parents=True) + + monkeypatch.setattr(backends.VirtualenvBackend, "perform_create", fake_create) + monkeypatch.setattr(backends.VenvBackend, "perform_create", fake_create) + monkeypatch.setattr(backends.CondaBackend, "perform_create", fake_create) + + +@pytest.mark.usefixtures("fake_create") +def test_venv_create(invoke, project): + project.project_config["venv.in_project"] = False + result = invoke(["venv", "create"], obj=project) + assert result.exit_code == 0, result.stderr + venv_path = re.match( + r"Virtualenv (.+) is created successfully", result.output + ).group(1) + assert os.path.exists(venv_path) + + +@pytest.mark.usefixtures("fake_create") +def test_venv_create_in_project(invoke, project): + project.project_config["venv.in_project"] = True + invoke(["venv", "create"], obj=project, strict=True) + invoke(["venv", "create"], obj=project, strict=True) + venv_path = project.root / ".venv" + assert venv_path.exists() + assert len(os.listdir(project.root / "venvs")) == 1 + + +@pytest.mark.usefixtures("fake_create") +def test_venv_list(invoke, project): + project.project_config["venv.in_project"] = False + result = invoke(["venv", "create"], obj=project) + assert result.exit_code == 0, result.stderr + venv_path = re.match( + r"Virtualenv (.+) is created successfully", result.output + ).group(1) + + result = invoke(["venv", "list"], obj=project) + assert result.exit_code == 0, result.stderr + assert venv_path in result.output + + +@pytest.mark.usefixtures("fake_create") +def test_venv_remove(invoke, project): + project.project_config["venv.in_project"] = False + result = invoke(["venv", "create"], obj=project) + assert result.exit_code == 0, result.stderr + venv_path = re.match( + r"Virtualenv (.+) is created successfully", result.output + ).group(1) + key = os.path.basename(venv_path)[len(get_venv_prefix(project)) :] + + result = invoke(["venv", "remove", "non-exist"], obj=project) + assert result.exit_code != 0 + + result = invoke(["venv", "remove", "-y", key], obj=project) + assert result.exit_code == 0, result.stderr + + assert not os.path.exists(venv_path) + + +@pytest.mark.usefixtures("fake_create") +def test_venv_recreate(invoke, project): + project.project_config["venv.in_project"] = False + result = invoke(["venv", "create"], obj=project) + assert result.exit_code == 0, result.stderr + + result = invoke(["venv", "create"], obj=project) + assert result.exit_code != 0 + + result = invoke(["venv", "create", "-f"], obj=project) + assert result.exit_code == 0, result.stderr + + +@pytest.mark.usefixtures("venv_backends") +def test_venv_activate(invoke, mocker, project): + project.project_config["venv.in_project"] = False + result = invoke(["venv", "create"], obj=project) + assert result.exit_code == 0, result.stderr + venv_path = re.match( + r"Virtualenv (.+) is created successfully", result.output + ).group(1) + key = os.path.basename(venv_path)[len(get_venv_prefix(project)) :] + + mocker.patch("shellingham.detect_shell", return_value=("bash", None)) + result = invoke(["venv", "activate", key], obj=project) + assert result.exit_code == 0, result.stderr + backend = project.config["venv.backend"] + + if backend == "conda": + assert result.output.startswith("conda activate") + else: + assert result.output.strip("'\"\n").endswith("activate") + assert result.output.startswith("source") + + +@pytest.mark.usefixtures("fake_create") +def test_venv_activate_error(invoke, project): + project.project_config["venv.in_project"] = False + result = invoke(["venv", "create"], obj=project, strict=True) + + result = invoke(["venv", "activate", "foo"], obj=project) + assert result.exit_code != 0 + assert "No virtualenv with key" in result.stderr + + result = invoke(["venv", "activate"], obj=project) + assert result.exit_code != 0 + assert "Can't activate a non-venv Python" in result.stderr + + +@pytest.mark.usefixtures("fake_create") +@pytest.mark.parametrize("keep_pypackages", [True, False]) +def test_venv_auto_create(invoke, mocker, project, keep_pypackages): + creator = mocker.patch("pdm.cli.commands.venv.backends.Backend.create") + del project.project_config["python.path"] + if keep_pypackages: + project.root.joinpath("__pypackages__").mkdir(exist_ok=True) + else: + shutil.rmtree(project.root / "__pypackages__", ignore_errors=True) + project.project_config["python.use_venv"] = True + invoke(["install"], obj=project) + if keep_pypackages: + creator.assert_not_called() + else: + creator.assert_called_once() + + +@pytest.mark.usefixtures("fake_create") +def test_venv_purge(invoke, project): + project.project_config["venv.in_project"] = False + result = invoke(["venv", "purge"], obj=project) + assert result.exit_code == 0, result.stderr + + result = invoke(["venv", "create"], obj=project) + assert result.exit_code == 0, result.stderr + venv_path = re.match( + r"Virtualenv (.+) is created successfully", result.output + ).group(1) + result = invoke(["venv", "purge"], input="y", obj=project) + assert result.exit_code == 0, result.stderr + assert not os.path.exists(venv_path) + + +@pytest.mark.usefixtures("fake_create") +def test_venv_purge_force(invoke, project): + project.project_config["venv.in_project"] = False + result = invoke(["venv", "create"], obj=project) + assert result.exit_code == 0, result.stderr + venv_path = re.match( + r"Virtualenv (.+) is created successfully", result.output + ).group(1) + result = invoke(["venv", "purge", "-f"], obj=project) + assert result.exit_code == 0, result.stderr + assert not os.path.exists(venv_path) + + +user_options = [("none", True), ("0", False), ("all", False)] + + +@pytest.mark.usefixtures("venv_backends") +@pytest.mark.parametrize("user_choices, is_path_exists", user_options) +def test_venv_purge_interactive(invoke, user_choices, is_path_exists, project): + project.project_config["venv.in_project"] = False + result = invoke(["venv", "create"], obj=project) + assert result.exit_code == 0, result.stderr + venv_path = re.match( + r"Virtualenv (.+) is created successfully", result.output + ).group(1) + result = invoke(["venv", "purge", "-i"], input=user_choices, obj=project) + assert result.exit_code == 0, result.stderr + assert os.path.exists(venv_path) == is_path_exists + + +def test_virtualenv_backend_create(project, mocker): + interpreter = project.python_executable + backend = backends.VirtualenvBackend(project, None) + assert backend.ident + mock_call = mocker.patch("subprocess.check_call") + location = backend.create() + mock_call.assert_called_once_with( + [ + sys.executable, + "-m", + "virtualenv", + "--no-pip", + "--no-setuptools", + "--no-wheel", + str(location), + "-p", + interpreter, + ], + stdout=ANY, + ) + + +def test_venv_backend_create(project, mocker): + interpreter = project.python_executable + backend = backends.VenvBackend(project, None) + assert backend.ident + mock_call = mocker.patch("subprocess.check_call") + location = backend.create() + mock_call.assert_called_once_with( + [interpreter, "-m", "venv", "--without-pip", str(location)], stdout=ANY + ) + + +def test_conda_backend_create(project, mocker): + backend = backends.CondaBackend(project, "3.8") + assert backend.ident == "3.8" + mock_call = mocker.patch("subprocess.check_call") + location = backend.create() + mock_call.assert_called_once_with( + ["conda", "create", "--yes", "--prefix", str(location), "python=3.8"], + stdout=ANY, + ) + + backend = backends.CondaBackend(project, None) + python_version = f"{sys.version_info.major}.{sys.version_info.minor}" + assert backend.ident.startswith(python_version) + location = backend.create() + mock_call.assert_called_with( + [ + "conda", + "create", + "--yes", + "--prefix", + str(location), + f"python={python_version}", + ], + stdout=ANY, + ) diff --git a/tests/conftest.py b/tests/conftest.py index 8f7af898..18b4b4dc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,7 +6,6 @@ import json import os import shutil import sys -from contextlib import contextmanager from dataclasses import dataclass from io import BytesIO, StringIO from pathlib import Path @@ -40,16 +39,6 @@ from tests import FIXTURES os.environ.update(CI="1", PDM_CHECK_UPDATE="0") -@contextmanager -def temp_environ(): - environ = os.environ.copy() - try: - yield - finally: - os.environ.clear() - os.environ.update(environ) - - class LocalFileAdapter(requests.adapters.BaseAdapter): def __init__(self, aliases, overrides=None, strip_suffix=False): super().__init__() @@ -269,6 +258,8 @@ def remove_pep582_path_from_pythonpath(pythonpath): @pytest.fixture() def core(): old_config_map = Config._config_map.copy() + # Turn off use_venv by default, for testing + Config._config_map["python.use_venv"].default = False main = Core() yield main # Restore the config items @@ -281,7 +272,7 @@ def index(): @pytest.fixture() -def project_no_init(tmp_path, mocker, core, index): +def project_no_init(tmp_path, mocker, core, index, monkeypatch): test_home = tmp_path / ".pdm-home" test_home.mkdir(parents=True) test_home.joinpath("config.toml").write_text( @@ -292,6 +283,7 @@ def project_no_init(tmp_path, mocker, core, index): p = core.create_project( tmp_path, global_config=test_home.joinpath("config.toml").as_posix() ) + p.global_config["venv.location"] = str(tmp_path / "venvs") mocker.patch( "pdm.models.environment.PDMSession", functools.partial(get_pypi_session, overrides=index), @@ -303,16 +295,15 @@ def project_no_init(tmp_path, mocker, core, index): getattr(sys, "_base_executable", sys.executable), HookManager(p, ["post_use"]), ) - with temp_environ(): - os.environ.pop("VIRTUAL_ENV", None) - os.environ.pop("CONDA_PREFIX", None) - os.environ.pop("PEP582_PACKAGES", None) - os.environ.pop("NO_SITE_PACKAGES", None) - pythonpath = os.environ.pop("PYTHONPATH", "") - pythonpath = remove_pep582_path_from_pythonpath(pythonpath) - if pythonpath: - os.environ["PYTHONPATH"] = pythonpath - yield p + monkeypatch.delenv("VIRTUAL_ENV", raising=False) + monkeypatch.delenv("CONDA_PREFIX", raising=False) + monkeypatch.delenv("PEP582_PACKAGES", raising=False) + monkeypatch.delenv("NO_SITE_PACKAGES", raising=False) + pythonpath = os.getenv("PYTHONPATH", "") + pythonpath = remove_pep582_path_from_pythonpath(pythonpath) + if pythonpath: + monkeypatch.setenv("PYTHONPATH", pythonpath) + yield p @pytest.fixture() @@ -447,3 +438,13 @@ def invoke(core, monkeypatch): return result return caller + + +BACKENDS = ["virtualenv", "venv"] + + +@pytest.fixture(params=BACKENDS) +def venv_backends(project, request): + project.project_config["venv.backend"] = request.param + project.project_config["python.use_venv"] = True + shutil.rmtree(project.root / "__pypackages__", ignore_errors=True) diff --git a/tests/test_project.py b/tests/test_project.py index 64e87cbd..bf41041c 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -6,14 +6,15 @@ from pathlib import Path import pytest from packaging.version import parse +from pdm.cli.commands.venv.utils import get_venv_python from pdm.utils import cd -def test_project_python_with_pyenv_support(project, mocker): +def test_project_python_with_pyenv_support(project, mocker, monkeypatch): del project.project_config["python.path"] project._python = None - os.environ["PDM_IGNORE_SAVED_PYTHON"] = "1" + monkeypatch.setenv("PDM_IGNORE_SAVED_PYTHON", "1") mocker.patch("pdm.project.core.PYENV_ROOT", str(project.root)) pyenv_python = project.root / "shims/python" if os.name == "nt": @@ -61,9 +62,9 @@ def test_project_sources_overriding(project): assert project.sources[0]["url"] == "https://example.org/simple" -def test_project_sources_env_var_expansion(project): - os.environ["PYPI_USER"] = "user" - os.environ["PYPI_PASS"] = "password" +def test_project_sources_env_var_expansion(project, monkeypatch): + monkeypatch.setenv("PYPI_USER", "user") + monkeypatch.setenv("PYPI_PASS", "password") project.project_config[ "pypi.url" ] = "https://${PYPI_USER}:${PYPI_PASS}@test.pypi.org/simple" @@ -162,13 +163,14 @@ def test_project_auto_detect_venv(project): assert project.environment.is_global -def test_ignore_saved_python(project): +@pytest.mark.path +def test_ignore_saved_python(project, monkeypatch): project.project_config["python.use_venv"] = True project._python = None scripts = "Scripts" if os.name == "nt" else "bin" suffix = ".exe" if os.name == "nt" else "" venv.create(project.root / "venv") - os.environ["PDM_IGNORE_SAVED_PYTHON"] = "1" + monkeypatch.setenv("PDM_IGNORE_SAVED_PYTHON", "1") assert project.python.executable != project.project_config["python.path"] assert ( project.python.executable == project.root / "venv" / scripts / f"python{suffix}" @@ -213,4 +215,53 @@ def test_global_python_path_config(project_no_init, tmp_path): def test_set_non_exist_python_path(project_no_init): project_no_init.project_config["python.path"] = "non-exist-python" project_no_init._python = None - assert project_no_init.python.executable == Path(sys.executable) + assert project_no_init.python.executable.name != "non-exist-python" + + +@pytest.mark.usefixtures("venv_backends") +def test_create_venv_first_time(invoke, project): + project.project_config.update({"venv.in_project": False}) + del project.project_config["python.path"] + result = invoke(["install"], obj=project) + assert result.exit_code == 0 + venv_parent = project.root / "venvs" + venv_path = next(venv_parent.iterdir(), None) + assert venv_path is not None + + assert Path(project.project_config["python.path"]).relative_to(venv_path) + + +@pytest.mark.usefixtures("venv_backends") +def test_create_venv_in_project(invoke, project): + project.project_config.update({"venv.in_project": True}) + del project.project_config["python.path"] + result = invoke(["install"], obj=project) + assert result.exit_code == 0 + assert project.root.joinpath(".venv").exists() + + +@pytest.mark.usefixtures("venv_backends") +def test_find_interpreters_from_venv(invoke, project): + project.project_config.update({"venv.in_project": False}) + del project.project_config["python.path"] + result = invoke(["install"], obj=project) + assert result.exit_code == 0 + venv_parent = project.root / "venvs" + venv_path = next(venv_parent.iterdir(), None) + venv_python = get_venv_python(venv_path) + + assert any(venv_python == p.executable for p in project.find_interpreters()) + + +def test_iter_project_venvs(project): + from pdm.cli.commands.venv import utils + + venv_parent = Path(project.config["venv.location"]) + venv_prefix = utils.get_venv_prefix(project) + for name in ("foo", "bar", "baz"): + venv_parent.joinpath(venv_prefix + name).mkdir(parents=True) + dot_venv_python = utils.get_venv_python(project.root / ".venv") + dot_venv_python.parent.mkdir(parents=True) + dot_venv_python.touch() + venv_keys = [key for key, _ in utils.iter_venvs(project)] + assert sorted(venv_keys) == ["bar", "baz", "foo", "in-project"] diff --git a/tests/test_utils.py b/tests/test_utils.py index a4dd95a9..4623078c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,4 @@ import pathlib -import re import sys import pytest @@ -50,6 +49,10 @@ def test_expend_env_vars_in_auth(given, expected, monkeypatch): assert utils.expand_env_vars_in_auth(given) == expected +def compare_python_paths(path1, path2): + return path1.parent == path2.parent + + @pytest.mark.path def test_find_python_in_path(tmp_path): @@ -58,17 +61,10 @@ def test_find_python_in_path(tmp_path): == pathlib.Path(sys.executable).resolve() ) - posix_path_to_executable = pathlib.Path(sys.executable).as_posix().lower() - if sys.platform == "darwin": - found_version_of_executable = re.split( - r"(python@[\d.]*\d+)", posix_path_to_executable - ) - posix_path_to_executable = "".join(found_version_of_executable[0:2]) - assert ( - utils.find_python_in_path(sys.prefix) - .as_posix() - .lower() - .startswith(posix_path_to_executable) + posix_path_to_executable = pathlib.Path(sys.executable) + assert compare_python_paths( + utils.find_python_in_path(sys.prefix), + posix_path_to_executable, ) assert not utils.find_python_in_path(tmp_path) -- GitLab