Skip to content

Commit

Permalink
pythonPackages: ensure all derivations provide python modules
Browse files Browse the repository at this point in the history 
This adds a test to ensure no new uses of `buildPythonApplication` can
be added to `python-packages.nix`.

Python packages can be grouped into two groups: 1) applications and 2)
packages providing importable modules. In `python-packages.nix` we only
want to have 2). 1) should be in the top-level package set.

To achieve this, all setup hooks need to be marked as being a setup hook.
For the setup hooks in the Python packages set this is done by creating
a new builder, `makePythonHook`.

Because there were issues with splicing, the file importing all the hooks
is converted to an extension. All non-packages were moved out of `python-packages.nix`
into `python-packages-base.nix`. The `keep` argument to `makeScopeWithSplicing
was cleaned up as well; there is no need to keep this one manually in sync
reducing the risk of breaking cross-compilation.
  • Loading branch information
@FRidh
FRidh committed Oct 27, 2022
1 parent 99bcead commit 33d12e5
Show file tree
Hide file tree
Showing 6 changed files with 185 additions and 242 deletions.
62 changes: 20 additions & 42 deletions pkgs/development/interpreters/python/default.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,19 @@
, pythonAttr ? null
, self # is pythonOnHostForTarget
}: let
pythonPackages = callPackage
pythonPackages = let
ensurePythonModules = items: let
providesSetupHook = lib.attrByPath [ "provides" "setupHook"] false;
notValid = value: (lib.isDerivation value) && !((pythonPackages.hasPythonModule value) || (providesSetupHook value));
func = name: value: if !(notValid value) then value else throw "${name} should use `buildPythonPackage` or `toPythonModule` if it is to be part of the Python packages set.";
in lib.mapAttrs func items;
in ensurePythonModules (callPackage
# Function that when called
# - imports python-packages.nix
# - adds spliced package sets to the package set
# - applies overrides from `packageOverrides` and `pythonPackagesOverlays`.
({ pkgs, stdenv, python, overrides }: let
pythonPackagesFun = import ../../../top-level/python-packages.nix {
pythonPackagesFun = import ./python-packages-base.nix {
inherit stdenv pkgs lib;
python = self;
};
Expand All @@ -48,59 +54,31 @@
selfHostHost = pythonOnHostForHost.pkgs;
selfTargetTarget = pythonOnTargetForTarget.pkgs or {}; # There is no Python TargetTarget.
};
keep = self: {
# TODO maybe only define these here so nothing is needed to be kept in sync.
inherit (self)
isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy310 isPy3k isPyPy pythonAtLeast pythonOlder
python bootstrapped-pip buildPythonPackage buildPythonApplication
fetchPypi
hasPythonModule requiredPythonModules makePythonPath disabledIf
toPythonModule toPythonApplication
buildSetupcfg

condaInstallHook
condaUnpackHook
eggUnpackHook
eggBuildHook
eggInstallHook
flitBuildHook
pipBuildHook
pipInstallHook
pytestCheckHook
pythonCatchConflictsHook
pythonImportsCheckHook
pythonNamespacesHook
pythonRecompileBytecodeHook
pythonRemoveBinBytecodeHook
pythonRemoveTestsDirHook
setuptoolsBuildHook
setuptoolsCheckHook
venvShellHook
wheelUnpackHook

wrapPython

pythonPackages

recursivePthLoader
;
};
hooks = import ./hooks/default.nix;
keep = lib.extends hooks pythonPackagesFun;
extra = _: {};
optionalExtensions = cond: as: if cond then as else [];
pythonExtension = import ../../../top-level/python-packages.nix;
python2Extension = import ../../../top-level/python2-packages.nix;
extensions = lib.composeManyExtensions ((optionalExtensions (!self.isPy3k) [python2Extension]) ++ pythonPackagesExtensions ++ [ overrides ]);
extensions = lib.composeManyExtensions ([
pythonExtension
] ++ (optionalExtensions (!self.isPy3k) [
python2Extension
]) ++ pythonPackagesExtensions ++ [
overrides
]);
aliases = self: super: lib.optionalAttrs config.allowAliases (import ../../../top-level/python-aliases.nix lib self super);
in lib.makeScopeWithSplicing
splicePackages
newScope
otherSplices
keep
extra
(lib.extends (lib.composeExtensions aliases extensions) pythonPackagesFun))
(lib.extends (lib.composeExtensions aliases extensions) keep))
{
overrides = packageOverrides;
python = self;
};
});
in rec {
isPy27 = pythonVersion == "2.7";
isPy35 = pythonVersion == "3.5";
Expand Down
114 changes: 56 additions & 58 deletions pkgs/development/interpreters/python/hooks/default.nix
View file
Original file line number Diff line number Diff line change
@@ -1,123 +1,117 @@
# Hooks for building Python packages.
{ python
, lib
, makeSetupHook
, disabledIf
, isPy3k
}:
self: super: with self;

let
callPackage = python.pythonForBuild.pkgs.callPackage;
pythonInterpreter = python.pythonForBuild.interpreter;
pythonSitePackages = python.sitePackages;
pythonCheckInterpreter = python.interpreter;
pythonInterpreter = super.python.pythonForBuild.interpreter;
pythonSitePackages = super.python.sitePackages;
pythonCheckInterpreter = super.python.interpreter;
setuppy = ../run_setup.py;
in rec {
in {
makePythonHook = args: pkgs.makeSetupHook ({passthru.provides.setupHook = true; } // args);

condaInstallHook = callPackage ({ gnutar, lbzip2 }:
makeSetupHook {
condaInstallHook = callPackage ({ makePythonHook, gnutar, lbzip2 }:
makePythonHook {
name = "conda-install-hook";
deps = [ gnutar lbzip2 ];
substitutions = {
inherit pythonSitePackages;
};
} ./conda-install-hook.sh) {};

condaUnpackHook = callPackage ({}:
makeSetupHook {
condaUnpackHook = callPackage ({ makePythonHook }:
makePythonHook {
name = "conda-unpack-hook";
deps = [];
} ./conda-unpack-hook.sh) {};

eggBuildHook = callPackage ({ }:
makeSetupHook {
eggBuildHook = callPackage ({ makePythonHook }:
makePythonHook {
name = "egg-build-hook.sh";
deps = [ ];
} ./egg-build-hook.sh) {};

eggInstallHook = callPackage ({ setuptools }:
makeSetupHook {
eggInstallHook = callPackage ({ makePythonHook, setuptools }:
makePythonHook {
name = "egg-install-hook.sh";
deps = [ setuptools ];
substitutions = {
inherit pythonInterpreter pythonSitePackages;
};
} ./egg-install-hook.sh) {};

eggUnpackHook = callPackage ({ }:
makeSetupHook {
eggUnpackHook = callPackage ({ makePythonHook, }:
makePythonHook {
name = "egg-unpack-hook.sh";
deps = [ ];
} ./egg-unpack-hook.sh) {};

flitBuildHook = callPackage ({ flit }:
makeSetupHook {
flitBuildHook = callPackage ({ makePythonHook, flit }:
makePythonHook {
name = "flit-build-hook";
deps = [ flit ];
substitutions = {
inherit pythonInterpreter;
};
} ./flit-build-hook.sh) {};

pipBuildHook = callPackage ({ pip, wheel }:
makeSetupHook {
pipBuildHook = callPackage ({ makePythonHook, pip, wheel }:
makePythonHook {
name = "pip-build-hook.sh";
deps = [ pip wheel ];
substitutions = {
inherit pythonInterpreter pythonSitePackages;
};
} ./pip-build-hook.sh) {};

pipInstallHook = callPackage ({ pip }:
makeSetupHook {
pipInstallHook = callPackage ({ makePythonHook, pip }:
makePythonHook {
name = "pip-install-hook";
deps = [ pip ];
substitutions = {
inherit pythonInterpreter pythonSitePackages;
};
} ./pip-install-hook.sh) {};

pytestCheckHook = callPackage ({ pytest }:
makeSetupHook {
pytestCheckHook = callPackage ({ makePythonHook, pytest }:
makePythonHook {
name = "pytest-check-hook";
deps = [ pytest ];
substitutions = {
inherit pythonCheckInterpreter;
};
} ./pytest-check-hook.sh) {};

pythonCatchConflictsHook = callPackage ({ setuptools }:
makeSetupHook {
pythonCatchConflictsHook = callPackage ({ makePythonHook, setuptools }:
makePythonHook {
name = "python-catch-conflicts-hook";
substitutions = {
inherit pythonInterpreter pythonSitePackages setuptools;
catchConflicts=../catch_conflicts/catch_conflicts.py;
};
} ./python-catch-conflicts-hook.sh) {};

pythonImportsCheckHook = callPackage ({}:
makeSetupHook {
pythonImportsCheckHook = callPackage ({ makePythonHook }:
makePythonHook {
name = "python-imports-check-hook.sh";
substitutions = {
inherit pythonCheckInterpreter;
};
} ./python-imports-check-hook.sh) {};

pythonNamespacesHook = callPackage ({ findutils }:
makeSetupHook {
pythonNamespacesHook = callPackage ({ makePythonHook, findutils }:
makePythonHook {
name = "python-namespaces-hook.sh";
substitutions = {
inherit pythonSitePackages findutils;
};
} ./python-namespaces-hook.sh) {};

pythonOutputDistHook = callPackage ({ }:
makeSetupHook {
pythonOutputDistHook = callPackage ({ makePythonHook }:
makePythonHook {
name = "python-output-dist-hook";
} ./python-output-dist-hook.sh ) {};

pythonRecompileBytecodeHook = callPackage ({ }:
makeSetupHook {
pythonRecompileBytecodeHook = callPackage ({ makePythonHook }:
makePythonHook {
name = "python-recompile-bytecode-hook";
substitutions = {
inherit pythonInterpreter pythonSitePackages;
Expand All @@ -126,71 +120,75 @@ in rec {
};
} ./python-recompile-bytecode-hook.sh ) {};

pythonRelaxDepsHook = callPackage ({ wheel }:
makeSetupHook {
pythonRelaxDepsHook = callPackage ({ makePythonHook, wheel }:
makePythonHook {
name = "python-relax-deps-hook";
deps = [ wheel ];
substitutions = {
inherit pythonInterpreter;
};
} ./python-relax-deps-hook.sh) {};

pythonRemoveBinBytecodeHook = callPackage ({ }:
makeSetupHook {
pythonRemoveBinBytecodeHook = callPackage ({ makePythonHook }:
makePythonHook {
name = "python-remove-bin-bytecode-hook";
} ./python-remove-bin-bytecode-hook.sh) {};

pythonRemoveTestsDirHook = callPackage ({ }:
makeSetupHook {
pythonRemoveTestsDirHook = callPackage ({ makePythonHook }:
makePythonHook {
name = "python-remove-tests-dir-hook";
substitutions = {
inherit pythonSitePackages;
};
} ./python-remove-tests-dir-hook.sh) {};

setuptoolsBuildHook = callPackage ({ setuptools, wheel }:
makeSetupHook {
setuptoolsBuildHook = callPackage ({ makePythonHook, setuptools, wheel }:
makePythonHook {
name = "setuptools-setup-hook";
deps = [ setuptools wheel ];
substitutions = {
inherit pythonInterpreter pythonSitePackages setuppy;
};
} ./setuptools-build-hook.sh) {};

setuptoolsCheckHook = callPackage ({ setuptools }:
makeSetupHook {
setuptoolsCheckHook = callPackage ({ makePythonHook, setuptools }:
makePythonHook {
name = "setuptools-check-hook";
deps = [ setuptools ];
substitutions = {
inherit pythonCheckInterpreter setuppy;
};
} ./setuptools-check-hook.sh) {};

unittestCheckHook = callPackage ({ }:
makeSetupHook {
unittestCheckHook = callPackage ({ makePythonHook }:
makePythonHook {
name = "unittest-check-hook";
substitutions = {
inherit pythonCheckInterpreter;
};
} ./unittest-check-hook.sh) {};

venvShellHook = disabledIf (!isPy3k) (callPackage ({ ensureNewerSourcesForZipFilesHook }:
makeSetupHook {
venvShellHook = disabledIf (!isPy3k) (callPackage ({ makePythonHook, ensureNewerSourcesForZipFilesHook }:
makePythonHook {
name = "venv-shell-hook";
deps = [ ensureNewerSourcesForZipFilesHook ];
substitutions = {
inherit pythonInterpreter;
};
} ./venv-shell-hook.sh) {});

wheelUnpackHook = callPackage ({ wheel }:
makeSetupHook {
wheelUnpackHook = callPackage ({ makePythonHook, wheel }:
makePythonHook {
name = "wheel-unpack-hook.sh";
deps = [ wheel ];
} ./wheel-unpack-hook.sh) {};

sphinxHook = callPackage ({ sphinx, installShellFiles }:
makeSetupHook {
wrapPython = callPackage ../wrap-python.nix {
inherit (pkgs.buildPackages) makeWrapper;
};

sphinxHook = callPackage ({ makePythonHook, sphinx, installShellFiles }:
makePythonHook {
name = "python${python.pythonVersion}-sphinx-hook";
deps = [ sphinx installShellFiles ];
} ./sphinx-hook.sh) {};
Expand Down
Loading

0 comments on commit 33d12e5

Please sign in to comment.