From f6467c357419d70d8f32816fe68b9bde6278f8b0 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 31 Aug 2023 22:41:09 +0200 Subject: [PATCH 1/3] pkgs/by-name: Introduce This introduces the `pkgs/by-name` directory as proposed by RFC 140. Included are: - The implementation to add packages defined in that directory to the top-level package scope - Contributer documentation on how to add packages to it - A GitHub Actions workflow to check the structure of it on all PRs --- .github/CODEOWNERS | 3 + .github/workflows/check-by-name.yml | 49 +++++++++++ pkgs/by-name/README.md | 101 ++++++++++++++++++++++ pkgs/test/nixpkgs-check-by-name/README.md | 3 +- pkgs/top-level/by-name-overlay.nix | 50 +++++++++++ pkgs/top-level/stage.nix | 8 ++ 6 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/check-by-name.yml create mode 100644 pkgs/by-name/README.md create mode 100644 pkgs/top-level/by-name-overlay.nix diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f5679a1a1420c63..79ba7b7c5200cd9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -49,6 +49,9 @@ # pkgs/by-name /pkgs/test/nixpkgs-check-by-name @infinisil +/pkgs/by-name/README.md @infinisil +/pkgs/top-level/by-name-overlay.nix @infinisil +/.github/workflows/check-by-name.nix @infinisil # Nixpkgs build-support /pkgs/build-support/writers @lassulus @Profpatsch diff --git a/.github/workflows/check-by-name.yml b/.github/workflows/check-by-name.yml new file mode 100644 index 000000000000000..9622634fcffd9c5 --- /dev/null +++ b/.github/workflows/check-by-name.yml @@ -0,0 +1,49 @@ +# Checks pkgs/by-name (see pkgs/by-name/README.md) +# using the nixpkgs-check-by-name tool (see pkgs/test/nixpkgs-check-by-name) +name: Check pkgs/by-name + +# The pre-built tool is fetched from a channel, +# making it work predictable on all PRs +on: pull_request + +# The tool doesn't need any permissions, it only outputs success or not based on the checkout +permissions: {} + +jobs: + check: + # This is x86_64-linux, for which the tool is always prebuilt on the nixos-* channels, + # as specified in nixos/release-combined.nix + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: cachix/install-nix-action@v22 + - name: Determining channel to use for dependencies + run: | + echo "Determining which channel to use for PR base branch $GITHUB_BASE_REF" + if [[ "$GITHUB_BASE_REF" =~ ^(release|staging|staging-next)-([0-9][0-9]\.[0-9][0-9])$ ]]; then + # Use the release channel for all PRs to release-XX.YY, staging-XX.YY and staging-next-XX.YY + channel=nixos-${BASH_REMATCH[2]} + echo "PR is for a release branch, using release channel $channel" + else + # Use the nixos-unstable channel for all other PRs + channel=nixos-unstable + echo "PR is for a non-release branch, using unstable channel $channel" + fi + echo "channel=$channel" >> "$GITHUB_ENV" + - name: Fetching latest version of channel + run: | + echo "Fetching latest version of channel $channel" + # This is probably the easiest way to get Nix to output the path to a downloaded channel! + nixpkgs=$(nix-instantiate --find-file nixpkgs -I nixpkgs=channel:"$channel") + # This file only exists in channels + rev=$(<"$nixpkgs"/.git-revision) + echo "Channel $channel is at revision $rev" + echo "nixpkgs=$nixpkgs" >> "$GITHUB_ENV" + echo "rev=$rev" >> "$GITHUB_ENV" + - name: Fetching pre-built nixpkgs-check-by-name from the channel + run: | + echo "Fetching pre-built nixpkgs-check-by-name from channel $channel at revision $rev" + # Passing --max-jobs 0 makes sure that we won't build anything + nix-build "$nixpkgs" -A tests.nixpkgs-check-by-name --max-jobs 0 + - name: Running nixpkgs-check-by-name + run: result/bin/nixpkgs-check-by-name . diff --git a/pkgs/by-name/README.md b/pkgs/by-name/README.md new file mode 100644 index 000000000000000..8aecc3822cc6153 --- /dev/null +++ b/pkgs/by-name/README.md @@ -0,0 +1,101 @@ +# Name-based package directories + +The structure of this directory maps almost directly to top-level package attributes. +This is the recommended way to add new top-level packages to Nixpkgs [when possible](#limitations). + +## Example + +The top-level package `pkgs.some-package` may be declared by setting up this file structure: + +``` +pkgs +└── by-name + ├── so + ┊ ├── some-package + ┊ └── package.nix + +``` + +Where `some-package` is the package name and `so` is the lowercased 2-letter prefix of the package name. + +The `package.nix` may look like this: + +```nix +# A function taking an attribute set as an argument +{ + # Get access to top-level attributes for use as dependencies + lib, + stdenv, + libbar, + + # Make this derivation configurable using `.override { enableBar = true }` + enableBar ? false, +}: + +# The return value must be a derivation +stdenv.mkDerivation { + # ... + buildInputs = + lib.optional enableBar libbar; +} +``` + +You can also split up the package definition into more files in the same directory if necessary. + +Once defined, the package can be built from the Nixpkgs root directory using: +``` +nix-build -A some-package +``` + +See the [general package conventions](../README.md#conventions) for more information on package definitions. + +### Changing implicit attribute defaults + +The above expression is called using these arguments by default: +```nix +{ + lib = pkgs.lib; + stdenv = pkgs.stdenv; + libbar = pkgs.libbar; +} +``` + +But the package might need `pkgs.libbar_2` instead. +While the function could be changed to take `libbar_2` directly as an argument, +this would change the `.override` interface, breaking code like `.override { libbar = ...; }`. +So instead it is preferable to use the same generic parameter name `libbar` +and override its value in [`pkgs/top-level/all-packages.nix`](../top-level/all-packages.nix): + +```nix +libfoo = callPackage ../by-name/so/somePackage/package.nix { + libbar = libbar_2; +}; +``` + +## Limitations + +There's some limitations as to which packages can be defined using this structure: + +- Only packages defined using `pkgs.callPackage`. + This excludes packages defined using `pkgs.python3Packages.callPackage ...`. + + Instead use the [category hierarchy](../README.md#category-hierarchy) for such attributes. + +- Only top-level packages. + This excludes packages for other package sets like `pkgs.pythonPackages.*`. + + Refer to the definition and documentation of the respective package set to figure out how such packages can be declared. + +## Validation + +CI performs [certain checks](../test/nixpkgs-check-by-name/README.md#validity-checks) on the `pkgs/by-name` structure. +This is done using the [`nixpkgs-check-by-name` tool](../test/nixpkgs-check-by-name). +The version of this tool used is the one that corresponds to the NixOS channel of the PR base branch. +See [here](../../.github/workflows/check-by-name.yml) for details. + +The tool can be run locally using + +```bash +nix-build -A tests.nixpkgs-check-by-name +result/bin/nixpkgs-check-by-name . +``` diff --git a/pkgs/test/nixpkgs-check-by-name/README.md b/pkgs/test/nixpkgs-check-by-name/README.md index 754d0a2090dfaf4..4dd694acd2f0af5 100644 --- a/pkgs/test/nixpkgs-check-by-name/README.md +++ b/pkgs/test/nixpkgs-check-by-name/README.md @@ -1,11 +1,12 @@ # Nixpkgs pkgs/by-name checker This directory implements a program to check the [validity](#validity-checks) of the `pkgs/by-name` Nixpkgs directory once introduced. +It is being used by [this GitHub Actions workflow](../../../.github/workflows/check-by-name.yml). This is part of the implementation of [RFC 140](https://github.com/NixOS/rfcs/pull/140). ## API -This API may be changed over time if the CI making use of it is adjusted to deal with the change appropriately, see [Hydra builds](#hydra-builds). +This API may be changed over time if the CI workflow making use of it is adjusted to deal with the change appropriately. - Command line: `nixpkgs-check-by-name ` - Arguments: diff --git a/pkgs/top-level/by-name-overlay.nix b/pkgs/top-level/by-name-overlay.nix new file mode 100644 index 000000000000000..41944c4e3b013ae --- /dev/null +++ b/pkgs/top-level/by-name-overlay.nix @@ -0,0 +1,50 @@ +# This file turns the pkgs/by-name directory (see its README.md for more info) into an overlay that adds all the defined packages. +# No validity checks are done here, +# instead this file is optimised for performance, +# and validity checks are done by CI on PRs. + +# Type: Path -> Overlay +baseDirectory: +let + # Because of Nix's import-value cache, importing lib is free + lib = import ../../lib; + + inherit (builtins) + readDir + ; + + inherit (lib.attrsets) + mapAttrs + mapAttrsToList + mergeAttrsList + ; + + # Package files for a single shard + # Type: String -> String -> AttrsOf Path + namesForShard = shard: type: + if type != "directory" then + # Ignore all non-directories. Technically only README.md is allowed as a file in the base directory, so we could alternatively: + # - Assume that README.md is the only file and change the condition to `shard == "README.md"` for a minor performance improvement. + # This would however cause very poor error messages if there's other files. + # - Ensure that README.md is the only file, throwing a better error message if that's not the case. + # However this would make for a poor code architecture, because one type of error would have to be duplicated in the validity checks and here. + # Additionally in either of those alternatives, we would have to duplicate the hardcoding of "README.md" + { } + else + mapAttrs + (name: _: baseDirectory + "/${shard}/${name}/package.nix") + (readDir (baseDirectory + "/${shard}")); + + # The attribute set mapping names to the package files defining them + # This is defined up here in order to allow reuse of the value (it's kind of expensive to compute) + # if the overlay has to be applied multiple times + packageFiles = mergeAttrsList (mapAttrsToList namesForShard (readDir baseDirectory)); +in +# TODO: Consider optimising this using `builtins.deepSeq packageFiles`, +# which could free up the above thunks and reduce GC times. +# Currently this would be hard to measure until we have more packages +# and ideally https://github.com/NixOS/nix/pull/8895 +self: super: +mapAttrs (name: file: + self.callPackage file { } +) packageFiles diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix index 3886ae04e492a65..1f37bbb70bda648 100644 --- a/pkgs/top-level/stage.nix +++ b/pkgs/top-level/stage.nix @@ -8,6 +8,13 @@ arguments. Normal users should not import this directly but instead import `pkgs/default.nix` or `default.nix`. */ +let + # An overlay to auto-call packages in ../by-name. + # By defining it at the top of the file, + # this value gets reused even if this file is imported multiple times, + # thanks to Nix's import-value cache. + autoCalledPackages = import ./by-name-overlay.nix ../by-name; +in { ## Misc parameters kept the same for all stages ## @@ -279,6 +286,7 @@ let stdenvAdapters trivialBuilders splice + autoCalledPackages allPackages otherPackageSets aliases From 87f7f1641ce421870d72a5fabaa934fe5de44c2d Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 31 Aug 2023 22:45:00 +0200 Subject: [PATCH 2/3] pkgs/README.md: Update to mention pkgs/by-name --- pkgs/README.md | 46 +++++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/pkgs/README.md b/pkgs/README.md index 2f486dc7a992f59..3fd3098f3b7059f 100644 --- a/pkgs/README.md +++ b/pkgs/README.md @@ -7,7 +7,7 @@ See the [CONTRIBUTING.md](../CONTRIBUTING.md) document for more general informat - [`top-level`](./top-level): Entrypoints, package set aggregations - [`impure.nix`](./top-level/impure.nix), [`default.nix`](./top-level/default.nix), [`config.nix`](./top-level/config.nix): Definitions for the evaluation entry point of `import ` - - [`stage.nix`](./top-level/stage.nix), [`all-packages.nix`](./top-level/all-packages.nix), [`splice.nix`](./top-level/splice.nix): Definitions for the top-level attribute set made available through `import {…}` + - [`stage.nix`](./top-level/stage.nix), [`all-packages.nix`](./top-level/all-packages.nix), [`by-name-overlay.nix`](./top-level/by-name-overlay.nix), [`splice.nix`](./top-level/splice.nix): Definitions for the top-level attribute set made available through `import {…}` - `*-packages.nix`, [`linux-kernels.nix`](./top-level/linux-kernels.nix), [`unixtools.nix`](./top-level/unixtools.nix): Aggregations of nested package sets defined in `development` - [`aliases.nix`](./top-level/aliases.nix), [`python-aliases.nix`](./top-level/python-aliases.nix): Aliases for package definitions that have been renamed or removed - `release*.nix`, [`make-tarball.nix`](./top-level/make-tarball.nix), [`packages-config.nix`](./top-level/packages-config.nix), [`metrics.nix`](./top-level/metrics.nix), [`nixpkgs-basic-release-checks.nix`](./top-level/nixpkgs-basic-release-checks.nix): Entry-points and utilities used by Hydra for continuous integration @@ -19,6 +19,7 @@ See the [CONTRIBUTING.md](../CONTRIBUTING.md) document for more general informat - [`stdenv`](./stdenv): [Standard environment](https://nixos.org/manual/nixpkgs/stable/#part-stdenv) - [`pkgs-lib`](./pkgs-lib): Definitions for utilities that need packages but are not needed for packages - [`test`](./test): Tests not directly associated with any specific packages +- [`by-name`](./by-name): Top-level packages organised by name ([docs](./by-name/README.md)) - All other directories loosely categorise top-level packages definitions, see [category hierarchy][categories] ## Quick Start to Adding a Package @@ -49,20 +50,23 @@ Now that this is out of the way. To add a package to Nixpkgs: $ cd nixpkgs ``` -2. Find a good place in the Nixpkgs tree to add the Nix expression for your package. For instance, a library package typically goes into `pkgs/development/libraries/pkgname`, while a web browser goes into `pkgs/applications/networking/browsers/pkgname`. See the [category hierarchy section][categories] for some hints on the tree organisation. Create a directory for your package, e.g. +2. Create a package directory `pkgs/by-name/so/some-package` where `some-package` is the package name and `so` is the lowercased 2-letter prefix of the package name: ```ShellSession - $ mkdir pkgs/development/libraries/libfoo + $ mkdir -p pkgs/by-name/so/some-package ``` -3. In the package directory, create a Nix expression — a piece of code that describes how to build the package. In this case, it should be a _function_ that is called with the package dependencies as arguments, and returns a build of the package in the Nix store. The expression should usually be called `default.nix`. + For more detailed information, see [here](./by-name/README.md). + +3. Create a `package.nix` file in the package directory, containing a Nix expression — a piece of code that describes how to build the package. In this case, it should be a _function_ that is called with the package dependencies as arguments, and returns a build of the package in the Nix store. ```ShellSession - $ emacs pkgs/development/libraries/libfoo/default.nix - $ git add pkgs/development/libraries/libfoo/default.nix + $ emacs pkgs/by-name/so/some-package/package.nix + $ git add pkgs/by-name/so/some-package/package.nix ``` - You can have a look at the existing Nix expressions under `pkgs/` to see how it’s done. Here are some good ones: + You can have a look at the existing Nix expressions under `pkgs/` to see how it’s done, some of which are also using the [category hierarchy](#category-hierarchy). + Here are some good ones: - GNU Hello: [`pkgs/applications/misc/hello/default.nix`](applications/misc/hello/default.nix). Trivial package, which specifies some `meta` attributes which is good practice. @@ -94,21 +98,13 @@ Now that this is out of the way. To add a package to Nixpkgs: The exact syntax and semantics of the Nix expression language, including the built-in function, are [described in the Nix manual](https://nixos.org/manual/nix/stable/language/). -4. Add a call to the function defined in the previous step to [`pkgs/top-level/all-packages.nix`](top-level/all-packages.nix) with some descriptive name for the variable, e.g. `libfoo`. - - ```ShellSession - $ emacs pkgs/top-level/all-packages.nix - ``` - - The attributes in that file are sorted by category (like “Development / Libraries”) that more-or-less correspond to the directory structure of Nixpkgs, and then by attribute name. - 5. To test whether the package builds, run the following command from the root of the nixpkgs source tree: ```ShellSession - $ nix-build -A libfoo + $ nix-build -A some-package ``` - where `libfoo` should be the variable name defined in the previous step. You may want to add the flag `-K` to keep the temporary build directory in case something fails. If the build succeeds, a symlink `./result` to the package in the Nix store is created. + where `some-package` should be the package name. You may want to add the flag `-K` to keep the temporary build directory in case something fails. If the build succeeds, a symlink `./result` to the package in the Nix store is created. 6. If you want to install the package into your profile (optional), do @@ -121,9 +117,19 @@ Now that this is out of the way. To add a package to Nixpkgs: ## Category Hierarchy [categories]: #category-hierarchy -Each package should be stored in its own directory somewhere in the `pkgs/` tree, i.e. in `pkgs/category/subcategory/.../pkgname`. Below are some rules for picking the right category for a package. Many packages fall under several categories; what matters is the _primary_ purpose of a package. For example, the `libxml2` package builds both a library and some tools; but it’s a library foremost, so it goes under `pkgs/development/libraries`. +Most top-level packages are organised in a loosely-categorised directory hierarchy in this directory. +See the [overview](#overview) for which directories are part of this. + +This category hierarchy is partially deprecated and will be migrated away over time. +The new `pkgs/by-name` directory ([docs](./by-name/README.md)) should be preferred instead. +The category hierarchy may still be used for packages that should be imported using an alternate `callPackage`, such as `python3Packages.callPackage` or `libsForQt5.callPackage`. -When in doubt, consider refactoring the `pkgs/` tree, e.g. creating new categories or splitting up an existing category. +If that is the case for a new package, here are some rules for picking the right category. +Many packages fall under several categories; what matters is the _primary_ purpose of a package. +For example, the `libxml2` package builds both a library and some tools; but it’s a library foremost, so it goes under `pkgs/development/libraries`. + +
+Categories **If it’s used to support _software development_:** @@ -299,6 +305,8 @@ A (typically large) program with a distinct user interface, primarily used inter - `misc` +
+ # Conventions ## Package naming From 77d50b03e4388f22e1f36a2621a9287a12a138be Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 31 Aug 2023 22:45:21 +0200 Subject: [PATCH 3/3] hello: Move to pkgs/by-name --- pkgs/README.md | 2 +- .../misc/hello/default.nix => by-name/he/hello/package.nix} | 0 pkgs/{applications/misc => by-name/he}/hello/test.nix | 0 pkgs/top-level/all-packages.nix | 2 -- 4 files changed, 1 insertion(+), 3 deletions(-) rename pkgs/{applications/misc/hello/default.nix => by-name/he/hello/package.nix} (100%) rename pkgs/{applications/misc => by-name/he}/hello/test.nix (100%) diff --git a/pkgs/README.md b/pkgs/README.md index 3fd3098f3b7059f..8833f7d54acab00 100644 --- a/pkgs/README.md +++ b/pkgs/README.md @@ -68,7 +68,7 @@ Now that this is out of the way. To add a package to Nixpkgs: You can have a look at the existing Nix expressions under `pkgs/` to see how it’s done, some of which are also using the [category hierarchy](#category-hierarchy). Here are some good ones: - - GNU Hello: [`pkgs/applications/misc/hello/default.nix`](applications/misc/hello/default.nix). Trivial package, which specifies some `meta` attributes which is good practice. + - GNU Hello: [`pkgs/by-name/he/hello/package.nix`](./by-name/he/hello/package.nix). Trivial package, which specifies some `meta` attributes which is good practice. - GNU cpio: [`pkgs/tools/archivers/cpio/default.nix`](tools/archivers/cpio/default.nix). Also a simple package. The generic builder in `stdenv` does everything for you. It has no dependencies beyond `stdenv`. diff --git a/pkgs/applications/misc/hello/default.nix b/pkgs/by-name/he/hello/package.nix similarity index 100% rename from pkgs/applications/misc/hello/default.nix rename to pkgs/by-name/he/hello/package.nix diff --git a/pkgs/applications/misc/hello/test.nix b/pkgs/by-name/he/hello/test.nix similarity index 100% rename from pkgs/applications/misc/hello/test.nix rename to pkgs/by-name/he/hello/test.nix diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 292624c41fd3586..b015b3b37f33867 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -32514,8 +32514,6 @@ with pkgs; heimer = libsForQt5.callPackage ../applications/misc/heimer { }; - hello = callPackage ../applications/misc/hello { }; - hello-wayland = callPackage ../applications/graphics/hello-wayland { }; hello-unfree = callPackage ../applications/misc/hello-unfree { };