From 4b89899dcc64c0b44c0068d34520d6cf1982c16d Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <silvan.mosberger@tweag.io>
Date: Wed, 23 Aug 2023 04:36:47 +0200
Subject: [PATCH] nixos/release-combined.nix: Build pkgs/by-name tester

---
 nixos/release-combined.nix                |  5 +++++
 pkgs/test/nixpkgs-check-by-name/README.md | 17 ++++++++++++++++-
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/nixos/release-combined.nix b/nixos/release-combined.nix
index 125086294d4108a..ffc22438b86456e 100644
--- a/nixos/release-combined.nix
+++ b/nixos/release-combined.nix
@@ -158,6 +158,11 @@ in rec {
         (onFullSupported "nixpkgs.emacs")
         (onFullSupported "nixpkgs.jdk")
         ["nixpkgs.tarball"]
+
+        # Ensure that nixpkgs-check-by-name is available in all release channels and nixos-unstable,
+        # so that a pre-built version can be used in CI for PR's on the corresponding development branches.
+        # See ../pkgs/test/nixpkgs-check-by-name/README.md
+        ["nixpkgs.tests.nixpkgs-check-by-name.x86_64-linux"]
       ];
     };
 }
diff --git a/pkgs/test/nixpkgs-check-by-name/README.md b/pkgs/test/nixpkgs-check-by-name/README.md
index 16536154a263093..754d0a2090dfaf4 100644
--- a/pkgs/test/nixpkgs-check-by-name/README.md
+++ b/pkgs/test/nixpkgs-check-by-name/README.md
@@ -5,7 +5,7 @@ This is part of the implementation of [RFC 140](https://github.com/NixOS/rfcs/pu
 
 ## API
 
-This API may be changed over time if the CI making use of it is adjusted to deal with the change appropriately.
+This API may be changed over time if the CI making use of it is adjusted to deal with the change appropriately, see [Hydra builds](#hydra-builds).
 
 - Command line: `nixpkgs-check-by-name <NIXPKGS>`
 - Arguments:
@@ -80,3 +80,18 @@ Tests are declared in [`./tests`](./tests) as subdirectories imitating Nixpkgs w
 - `expected` (optional):
   A file containing the expected standard output.
   The default is expecting an empty standard output.
+
+## Hydra builds
+
+This program will always be available pre-built for `x86_64-linux` on the `nixos-unstable` channel and `nixos-XX.YY` channels.
+This is ensured by including it in the `tested` jobset description in [`nixos/release-combined.nix`](../../../nixos/release-combined.nix).
+
+This allows CI for PRs to development branches `master` and `release-XX.YY` to fetch the pre-built program from the corresponding channel and use that to check the PR. This has the following benefits:
+- It allows CI to check all PRs, even if they would break the CI tooling.
+- It makes the CI check very fast, since no Nix builds need to be done, even for mass rebuilds.
+- It improves security, since we don't have to build potentially untrusted code from PRs.
+  The tool only needs a very minimal Nix evaluation at runtime, which can work with [readonly-mode](https://nixos.org/manual/nix/stable/command-ref/opt-common.html#opt-readonly-mode) and [restrict-eval](https://nixos.org/manual/nix/stable/command-ref/conf-file.html#conf-restrict-eval).
+- It allows anybody to make updates to the tooling and for those updates to be automatically used by CI without needing a separate release mechanism.
+
+The tradeoff is that there's a delay between updates to the tool and those updates being used by CI.
+This needs to be considered when updating the [API](#api).