Add how-to for using custom Python distributions (#1534)

docs/history/hatch.md

@@ -15,13 +15,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ***Added:***
 
 - Add ability to control the source of Python distributions
+- Upgrade PyApp to 0.22.0 for binary builds
 
 ***Fixed:***
 
 - The `fmt` command no longer hides the commands that are being executed
 - Add default timeout for network requests, useful when installing Python distributions
 - Fix syntax highlighting contrast for the `config show` command
-- Upgrade PyApp to 0.22.0 for binary builds
 
 ## [1.11.1](https://github.com/pypa/hatch/releases/tag/hatch-v1.11.1) - 2024-05-23 ## {: #hatch-v1.11.1 }
 

docs/how-to/python/custom.md

@@ -0,0 +1,30 @@
+# How to use custom Python distributions
+
+----
+
+The built-in [Python management](../../tutorials/python/manage.md) capabilities offer full support for using custom distributions.
+
+## Configuration
+
+Configuring custom Python distributions is done entirely through three environment variables that must all be defined, for each desired distribution. In the following sections, the placeholder `<NAME>` is the uppercased version of the distribution name with periods replaced by underscores e.g. `pypy3.10` would become `PYPY3_10`.
+
+### Source
+
+The `HATCH_PYTHON_CUSTOM_SOURCE_<NAME>` variable is the URL to the distribution's archive. The value must end with the archive's real file extension, which is used to determine the extraction method.
+
+The following extensions are supported:
+
+| Extensions | Description |
+| --- | --- |
+| <ul><li><code>.tar.bz2</code></li><li><code>.bz2</code></li></ul> | A [tar file](https://en.wikipedia.org/wiki/Tar_(computing)) with [bzip2 compression](https://en.wikipedia.org/wiki/Bzip2) |
+| <ul><li><code>.tar.gz</code></li><li><code>.tgz</code></li></ul> | A [tar file](https://en.wikipedia.org/wiki/Tar_(computing)) with [gzip compression](https://en.wikipedia.org/wiki/Gzip) |
+| <ul><li><code>.tar.zst</code></li><li><code>.tar.zstd</code></li></ul> | A [tar file](https://en.wikipedia.org/wiki/Tar_(computing)) with [Zstandard compression](https://en.wikipedia.org/wiki/Zstd) |
+| <ul><li><code>.zip</code></li></ul> | A [ZIP file](https://en.wikipedia.org/wiki/ZIP_(file_format)) with [DEFLATE compression](https://en.wikipedia.org/wiki/Deflate) |
+
+### Python path
+
+The `HATCH_PYTHON_CUSTOM_PATH_<NAME>` variable is the path to the Python interpreter within the archive. This path is relative to the root of the archive and must be a Unix-style path, even on Windows.
+
+### Version
+
+The `HATCH_PYTHON_CUSTOM_VERSION_<NAME>` variable is the version of the distribution. This value is used to determine whether updates are required and is displayed in the output of the [`python show`](../../cli/reference.md#hatch-python-show) command.

mkdocs.yml

@@ -131,11 +131,13 @@ nav:
       - Dependency resolution: how-to/environment/dependency-resolution.md
     - Static analysis:
       - Customize behavior: how-to/static-analysis/behavior.md
-    - Plugins:
-      - Testing builds: how-to/plugins/testing-builds.md
+    - Python:
+      - Custom distributions: how-to/python/custom.md
     - Publishing:
         - Authentication: how-to/publish/auth.md
         - Repository selection: how-to/publish/repo.md
+    - Plugins:
+      - Testing builds: how-to/plugins/testing-builds.md
   - Tutorials:
     - Python:
       - Management: tutorials/python/manage.md

src/hatch/config/constants.py

@@ -30,4 +30,6 @@ class PublishEnvVars:
 
 
 class PythonEnvVars:
-    SOURCE_PREFIX = 'HATCH_PYTHON_SOURCE_'
+    CUSTOM_SOURCE_PREFIX = 'HATCH_PYTHON_CUSTOM_SOURCE_'
+    CUSTOM_PATH_PREFIX = 'HATCH_PYTHON_CUSTOM_PATH_'
+    CUSTOM_VERSION_PREFIX = 'HATCH_PYTHON_CUSTOM_VERSION_'

src/hatch/python/resolve.py

@@ -17,6 +17,26 @@
     from hatch.utils.fs import Path
 
 
+# Use an artificially high epoch to ensure that custom distributions are always considered newer
+CUSTOM_DISTRIBUTION_VERSION_EPOCH = 100
+
+
+def custom_env_var(prefix: str, name: str) -> str:
+    return f'{prefix}{name.upper().replace(".", "_")}'
+
+
+def get_custom_source(name: str) -> str | None:
+    return os.environ.get(custom_env_var(PythonEnvVars.CUSTOM_SOURCE_PREFIX, name))
+
+
+def get_custom_version(name: str) -> str | None:
+    return os.environ.get(custom_env_var(PythonEnvVars.CUSTOM_VERSION_PREFIX, name))
+
+
+def get_custom_path(name: str) -> str | None:
+    return os.environ.get(custom_env_var(PythonEnvVars.CUSTOM_PATH_PREFIX, name))
+
+
 class Distribution(ABC):
     def __init__(self, name: str, source: str) -> None:
         self.__name = name
@@ -28,8 +48,7 @@ def name(self) -> str:
 
     @cached_property
     def source(self) -> str:
-        env_var = f'{PythonEnvVars.SOURCE_PREFIX}{self.name.upper().replace(".", "_")}'
-        return os.environ.get(env_var, self.__source)
+        return self.__source if (custom_source := get_custom_source(self.name)) is None else custom_source
 
     @cached_property
     def archive_name(self) -> str:
@@ -41,23 +60,23 @@ def unpack(self, archive: Path, directory: Path) -> None:
 
             with zipfile.ZipFile(archive, 'r') as zf:
                 zf.extractall(directory)
-        elif self.source.endswith('.tar.gz'):
+        elif self.source.endswith(('.tar.gz', '.tgz')):
             import tarfile
 
             with tarfile.open(archive, 'r:gz') as tf:
                 if sys.version_info[:2] >= (3, 12):
                     tf.extractall(directory, filter='data')
                 else:
                     tf.extractall(directory)  # noqa: S202
-        elif self.source.endswith('.tar.bz2'):
+        elif self.source.endswith(('.tar.bz2', '.bz2')):
             import tarfile
 
             with tarfile.open(archive, 'r:bz2') as tf:
                 if sys.version_info[:2] >= (3, 12):
                     tf.extractall(directory, filter='data')
                 else:
                     tf.extractall(directory)  # noqa: S202
-        elif self.source.endswith('.tar.zst'):
+        elif self.source.endswith(('.tar.zst', '.tar.zstd')):
             import tarfile
 
             import zstandard
@@ -89,6 +108,9 @@ class CPythonStandaloneDistribution(Distribution):
     def version(self) -> Version:
         from packaging.version import Version
 
+        if (custom_version := get_custom_version(self.name)) is not None:
+            return Version(f'{CUSTOM_DISTRIBUTION_VERSION_EPOCH}!{custom_version}')
+
         # .../cpython-3.12.0%2B20231002-...
         # .../cpython-3.7.9-...
         _, _, remaining = self.source.partition('/cpython-')
@@ -99,6 +121,9 @@ def version(self) -> Version:
 
     @cached_property
     def python_path(self) -> str:
+        if (custom_path := get_custom_path(self.name)) is not None:
+            return custom_path
+
         if self.name == '3.7':
             if sys.platform == 'win32':
                 return r'python\install\python.exe'
@@ -116,12 +141,18 @@ class PyPyOfficialDistribution(Distribution):
     def version(self) -> Version:
         from packaging.version import Version
 
+        if (custom_version := get_custom_version(self.name)) is not None:
+            return Version(f'{CUSTOM_DISTRIBUTION_VERSION_EPOCH}!{custom_version}')
+
         *_, remaining = self.source.partition('/pypy/')
         _, version, *_ = remaining.split('-')
         return Version(f'0!{version[1:]}')
 
     @cached_property
     def python_path(self) -> str:
+        if (custom_path := get_custom_path(self.name)) is not None:
+            return custom_path
+
         directory = self.archive_name
         for extension in ('.tar.bz2', '.zip'):
             if directory.endswith(extension):

tests/python/test_core.py

@@ -5,7 +5,7 @@
 from hatch.config.constants import PythonEnvVars
 from hatch.python.core import InstalledDistribution, PythonManager
 from hatch.python.distributions import ORDERED_DISTRIBUTIONS
-from hatch.python.resolve import get_distribution
+from hatch.python.resolve import custom_env_var, get_distribution
 from hatch.utils.structures import EnvVars
 
 
@@ -15,7 +15,7 @@ def test_custom_source(platform, current_arch, name):
         pytest.skip('No macOS 3.7 distribution for ARM')
 
     dist = get_distribution(name)
-    with EnvVars({f'{PythonEnvVars.SOURCE_PREFIX}{name.upper().replace(".", "_")}': 'foo'}):
+    with EnvVars({custom_env_var(PythonEnvVars.CUSTOM_SOURCE_PREFIX, name): 'foo'}):
         assert dist.source == 'foo'
 
 

tests/python/test_resolve.py

@@ -3,8 +3,9 @@
 
 import pytest
 
+from hatch.config.constants import PythonEnvVars
 from hatch.errors import PythonDistributionResolutionError, PythonDistributionUnknownError
-from hatch.python.resolve import get_distribution
+from hatch.python.resolve import custom_env_var, get_distribution
 from hatch.utils.structures import EnvVars
 
 
@@ -34,6 +35,15 @@ def test_cpython_standalone(self):
         assert version.epoch == 0
         assert version.base_version == '3.11.3'
 
+    def test_cpython_standalone_custom(self):
+        name = '3.11'
+        dist = get_distribution(name)
+        with EnvVars({custom_env_var(PythonEnvVars.CUSTOM_VERSION_PREFIX, name): '9000.42'}):
+            version = dist.version
+
+        assert version.epoch == 100
+        assert '.'.join(map(str, version.release)) == '9000.42'
+
     def test_pypy(self):
         url = 'https://downloads.python.org/pypy/pypy3.10-v7.3.12-aarch64.tar.bz2'
         dist = get_distribution('pypy3.10', url)
@@ -42,6 +52,29 @@ def test_pypy(self):
         assert version.epoch == 0
         assert version.base_version == '7.3.12'
 
+    def test_pypy_custom(self):
+        name = 'pypy3.10'
+        dist = get_distribution(name)
+        with EnvVars({custom_env_var(PythonEnvVars.CUSTOM_VERSION_PREFIX, name): '9000.42'}):
+            version = dist.version
+
+        assert version.epoch == 100
+        assert '.'.join(map(str, version.release)) == '9000.42'
+
+
+class TestDistributionPaths:
+    def test_cpython_standalone_custom(self):
+        name = '3.11'
+        dist = get_distribution(name)
+        with EnvVars({custom_env_var(PythonEnvVars.CUSTOM_PATH_PREFIX, name): 'foo/bar/python'}):
+            assert dist.python_path == 'foo/bar/python'
+
+    def test_pypy_custom(self):
+        name = 'pypy3.10'
+        dist = get_distribution(name)
+        with EnvVars({custom_env_var(PythonEnvVars.CUSTOM_PATH_PREFIX, name): 'foo/bar/python'}):
+            assert dist.python_path == 'foo/bar/python'
+
 
 @pytest.mark.parametrize(
     ('system', 'variant'),