Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support scripts with inline script metadata as input files #2107

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ project's virtual environment.

The `pip-compile` command lets you compile a `requirements.txt` file from
your dependencies, specified in either `pyproject.toml`, `setup.cfg`,
`setup.py`, or `requirements.in`.
`setup.py`, `requirements.in`, or pure-Python scripts containing
[inline script metadata](https://packaging.python.org/en/latest/specifications/inline-script-metadata/).

Run it with `pip-compile` or `python -m piptools compile` (or
`pipx run --spec pip-tools pip-compile` if `pipx` was installed with the
Expand Down
61 changes: 54 additions & 7 deletions piptools/scripts/compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import itertools
import os
import re
import shlex
import sys
import tempfile
Expand Down Expand Up @@ -33,6 +34,11 @@
from . import options
from .options import BuildTargetT

if sys.version_info >= (3, 11):
import tomllib
else:
import tomli as tomllib

DEFAULT_REQUIREMENTS_FILES = (
"requirements.in",
"setup.py",
Expand All @@ -43,6 +49,10 @@
DEFAULT_REQUIREMENTS_OUTPUT_FILE = "requirements.txt"
METADATA_FILENAMES = frozenset({"setup.py", "setup.cfg", "pyproject.toml"})

INLINE_SCRIPT_METADATA_REGEX = (
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I recommend using (?x) to be able to make this multiline with comments.

r"(?m)^# /// (?P<type>[a-zA-Z0-9-]+)$\s(?P<content>(^#(| .*)$\s)+)^# ///$"
)


def _determine_linesep(
strategy: str = "preserve", filenames: tuple[str, ...] = ()
Expand Down Expand Up @@ -170,7 +180,8 @@ def cli(
) -> None:
"""
Compiles requirements.txt from requirements.in, pyproject.toml, setup.cfg,
or setup.py specs.
or setup.py specs, as well as Python scripts containing inline script
metadata.
"""
if color is not None:
ctx.color = color
Expand Down Expand Up @@ -344,14 +355,50 @@ def cli(
)
raise click.BadParameter(msg)

if src_file == "-":
# pip requires filenames and not files. Since we want to support
# piping from stdin, we need to briefly save the input from stdin
# to a temporary file and have pip read that. also used for
if src_file == "-" or (
os.path.basename(src_file).endswith(".py") and not is_setup_file
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

pathlib is usually nicer

Suggested change
os.path.basename(src_file).endswith(".py") and not is_setup_file
Path(src_file).suffix == ".py" and not is_setup_file

):
# pip requires filenames and not files. Since we want to support
# piping from stdin, and inline script metadadata within Python
# scripts, we need to briefly save the input or extracted script
# dependencies to a temporary file and have pip read that. Also used for
# reading requirements from install_requires in setup.py.
if os.path.basename(src_file).endswith(".py"):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
if os.path.basename(src_file).endswith(".py"):
if Path(src_file).suffix == ".py":

# Probably contains inline script metadata
with open(src_file, encoding="utf-8") as f:
script = f.read()
name = "script"
matches = list(
filter(
lambda m: m.group("type") == name,
re.finditer(INLINE_SCRIPT_METADATA_REGEX, script),
)
)
if len(matches) > 1:
raise ValueError(f"Multiple {name} blocks found")
elif len(matches) == 1:
content = "".join(
line[2:] if line.startswith("# ") else line[1:]
for line in matches[0]
.group("content")
.splitlines(keepends=True)
)
metadata = tomllib.loads(content)
reqs_str = metadata.get("dependencies", [])
tmpfile = tempfile.NamedTemporaryFile(mode="wt", delete=False)
input_reqs = "\n".join(reqs_str)
comes_from = (
f"{os.path.basename(src_file)} (inline script metadata)"
)
else:
raise PipToolsError(
"Input script does not contain valid inline script metadata!"
)
else:
input_reqs = sys.stdin.read()
comes_from = "-r -"
tmpfile = tempfile.NamedTemporaryFile(mode="wt", delete=False)
tmpfile.write(sys.stdin.read())
comes_from = "-r -"
tmpfile.write(input_reqs)
tmpfile.flush()
reqs = list(
parse_requirements(
Expand Down
68 changes: 68 additions & 0 deletions tests/test_cli_compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from pip._vendor.packaging.version import Version

from piptools.build import ProjectMetadata
from piptools.exceptions import PipToolsError
from piptools.scripts.compile import cli
from piptools.utils import (
COMPILE_EXCLUDE_OPTIONS,
Expand Down Expand Up @@ -3771,3 +3772,70 @@ def test_stdout_should_not_be_read_when_stdin_is_not_a_plain_file(
out = runner.invoke(cli, [req_in.as_posix(), "--output-file", fifo.as_posix()])

assert out.exit_code == 0, out


def test_compile_inline_script_metadata(runner, tmp_path, current_resolver):
(tmp_path / "script.py").write_text(
dedent(
"""
# /// script
# dependencies = [
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we also have tests for extras?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, there's no explicit extras in the spec. But how about requires-python?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure how that would influence the build process?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It should reject incompatible runtimes, for example.

# "small-fake-with-deps",
# ]
# ///
"""
)
)
out = runner.invoke(
cli,
[
"--no-build-isolation",
"--no-header",
"--no-emit-options",
"--find-links",
os.fspath(MINIMAL_WHEELS_PATH),
os.fspath(tmp_path / "script.py"),
"--output-file",
"-",
],
)
expected = r"""small-fake-a==0.1
# via small-fake-with-deps
small-fake-with-deps==0.1
# via script.py (inline script metadata)
"""
assert out.exit_code == 0
assert expected == out.stdout


def test_compile_inline_script_metadata_invalid(runner, tmp_path, current_resolver):
(tmp_path / "script.py").write_text(
dedent(
"""
# /// invalid-name
# dependencies = [
# "small-fake-a",
# "small-fake-b",
# ]
# ///
"""
)
)
with pytest.raises(
PipToolsError, match="does not contain valid inline script metadata"
):
runner.invoke(
cli,
[
"--no-build-isolation",
"--no-header",
"--no-annotate",
"--no-emit-options",
"--find-links",
os.fspath(MINIMAL_WHEELS_PATH),
os.fspath(tmp_path / "script.py"),
"--output-file",
"-",
],
catch_exceptions=False,
)
Loading