treewide: Refactor to per-project structure

README.md

@@ -4,38 +4,49 @@
 of software projects funded by
 the [Next Generation Internet][NGI] (NGI) initiative of the European Commission
 
-[Nix]: https://nixos.org/manual/nix
-[NixOS]: https://nixos.org/manual/nixos
-[NGI]: https://www.ngi.eu
-
 ## Structure of NGIpkgs
 
 The software in NGIpkgs can be divided into two broad categories: Nix packages, and NixOS modules.
 
-Nix packages can theoretically be built and run on any operating system that runs the Nix package manager. The output of building a Nix package is often a usable library or executable and most if not all of its dependencies. In NGIpkgs, these packages are all contained in the `pkgs` directory. For simple package definitions, we use `pkgs/by-name/<pname>/package.nix`, inspired by [Nix RFC 140](https://github.com/NixOS/rfcs/blob/c8569f6719356009204133cd00d92010889ed56d/rfcs/0140-simple-package-paths.md). Otherwise, packages are added in `pkgs/<pname>/default.nix` imported in `pkgs/default.nix`.
+Nix packages can theoretically be built and run on any operating system that runs the Nix package manager.
+The output of building a Nix package is often a usable library or executable and most if not all of its dependencies.
+In NGIpkgs, these packages are all contained in the `pkgs` directory.
+For simple package definitions, we use `pkgs/by-name/<pname>/package.nix`, inspired by [Nix RFC 140][rfc-140].
+Otherwise, packages are added in `pkgs/<pname>/default.nix` imported in `pkgs/default.nix`.
+
+Corresponding to [funded projects](https://nlnet.nl/project/) there are per-project subdirectories within the `projects` directory.
+These per-project directories contain a `default.nix` which
+(a) picks packages associated with the project from those defined in `pkgs` and Nixpkgs,
+(b) exposes NixOS modules, tests and configurations which are also contained in the per-project directory,
+(c) may contain additional metadata about the project.
 
-NixOS modules are components that can be easily integrated into NixOS. Usually they enrich Nix packages with configuration parameters. Many of them represent services that map to one or more systemd service(s) that are designed to, run persistently on NixOS. These modules are defined in the `modules` directory of NGIpkgs, and they are ready to be deployed to a new NixOS system (such as a container, VM, or physical machine). Templates in `configs` are a good starting point for anyone interested in using modules, and they are also used for testing.
+NixOS modules are components that can be easily integrated into NixOS.
+Usually they enrich Nix packages with configuration parameters.
+Many of them represent services that map to one or more systemd service(s) that are designed to, run persistently on NixOS.
+These modules are ready to be deployed to a new NixOS system (such as a container, VM, or physical machine).
+Templates configurations found in the corresponding per-project directory are a good starting point for anyone interested in using modules, and they are also used for testing.
 
 ```
 .
 ├── flake.nix
 ├── pkgs
 │   ├── by-name
-│   │   └── ...           # directories of packages that are added `by-name`
-│   ├── default.nix       # imports all packages that are not in `by-name`
-│   └── ...               # directories for packages
-├── modules
-│   └── ...                       # add module files here
-├── README.md                     # this file
-├── configs
-│   ├── all-configurations.nix    # import configuration files here
-│   └── ...                       # add configuration directories here
-└── ...
+│   │   └── …            # directories of packages that are added `by-name`
+│   ├── default.nix      # imports all packages that are not in `by-name`
+│   └── …                # directories for packages
+├── projects
+│   ├── <project-name>   # names match with those at https://nlnet.nl/project
+│   │   ├── default.nix  # project definition
+│   │   └── …            # files of the project (e.g. NixOS module, configuration, tests, etc.)
+│   └── …
+├── README.md            # this file
+└── …
 ```
 
 ## Continuous Builds of Packages with Hydra
 
-All packages in the main branch of NGIpkgs are automatically built by a [Hydra](https://github.com/NixOS/hydra) server. The results of these builds can be seen at <https://hydra.ngi0.nixos.org/jobset/NGIpkgs/main#tabs-jobs>
+All packages in the main branch of NGIpkgs are automatically built by a [Hydra](https://github.com/NixOS/hydra) server.
+The results of these builds can be seen at <https://hydra.ngi0.nixos.org/jobset/NGIpkgs/main#tabs-jobs>
 
 ## Reasoning for Creation of the NGIpkgs Monorepo
 
@@ -46,3 +57,8 @@ All packages in the main branch of NGIpkgs are automatically built by a [Hydra](
 ## Contributing to NGIpkgs
 
 Please see [`CONTRIBUTING.md`](CONTRIBUTING.md)
+
+[Nix]: https://nixos.org/manual/nix
+[NixOS]: https://nixos.org/manual/nixos
+[NGI]: https://www.ngi.eu
+[rfc-140]: https://github.com/NixOS/rfcs/blob/c8569f6719356009204133cd00d92010889ed56d/rfcs/0140-simple-package-paths.md

flake.nix

@@ -27,12 +27,13 @@
     rust-overlay,
     dream2nix,
     ...
-  }: let
+  } @ inputs: let
     inherit
       (builtins)
-      readDir
       mapAttrs
       attrValues
+      isAttrs
+      concatStringsSep
       ;
 
     inherit
@@ -45,26 +46,51 @@
       nixosSystem
       filterAttrs
       attrByPath
+      foldlAttrs
       ;
 
+    flattenAttrs = {
+      prefix ? [],
+      sep ? ".",
+    }: x: let
+      f = path:
+        foldlAttrs (acc: name: value:
+          (
+            if isAttrs value
+            then (f "${path}${name}${sep}" value)
+            else {"${path}${name}" = value;}
+          )
+          // acc) {};
+    in
+      f (
+        if prefix == []
+        then ""
+        else (concatStringsSep sep prefix) + sep
+      )
+      x;
+
+    flattenAttrsSlash = flattenAttrs {sep = "/";};
+
+    importProjects = {
+      pkgs ? {},
+      lib ? inputs.nixpkgs.lib,
+      sources ? {
+        configurations = rawNixosConfigs;
+        modules = extendedModules;
+      },
+    }:
+      import ./projects {inherit lib pkgs sources;};
+
+    mapAttrByPath = attrPath: default: mapAttrs (_: attrByPath attrPath default);
+
+    pickNixosModules = mapAttrByPath ["nixos" "modules"] {};
+
+    pickNixosTests = mapAttrByPath ["nixos" "tests"] {};
+
+    pickNixosConfigurations = x: mapAttrs (_: v: mapAttrs (_: v: v.path) v) (mapAttrByPath ["nixos" "configurations"] {} x);
+
     importPackages = pkgs: let
-      nixosTests = let
-        dir = ./tests;
-        testDirs = readDir dir;
-
-        dirToTest = name: _: let
-          mkTestModule = import "${dir}/${name}";
-
-          testModule = mkTestModule {
-            inherit pkgs;
-            inherit (pkgs) lib;
-            modules = extendedModules;
-            configurations = rawNixosConfigs;
-          };
-        in
-          pkgs.nixosTest testModule;
-      in
-        mapAttrs dirToTest testDirs;
+      nixosTests = pickNixosTests (importProjects {pkgs = pkgs // allPackages;});
 
       callPackage = pkgs.newScope (
         allPackages // {inherit callPackage nixosTests;}
@@ -87,10 +113,13 @@
     importNixpkgs = system: overlays:
       import nixpkgs {inherit system overlays;};
 
-    rawNixosConfigs = import ./configs/all-configurations.nix;
+    rawNixosConfigs = flattenAttrsSlash (pickNixosConfigurations (importProjects {}));
 
     loadTreefmt = pkgs: treefmt-nix.lib.evalModule pkgs ./treefmt.nix;
 
+    # Overlays a package set (e.g. nixpkgs) with the packages defined in this flake.
+    overlay = final: prev: importPackages prev;
+
     # Attribute set containing all modules obtained via `inputs` and defined
     # in this flake towards definition of `nixosConfigurations` and `nixosTests`.
     extendedModules =
@@ -101,15 +130,17 @@
 
     nixosConfigurations =
       mapAttrs
-      (_: config: nixosSystem {modules = [config ./configs/dummy.nix] ++ attrValues extendedModules;})
+      (_: config: nixosSystem {modules = [config ./dummy.nix] ++ attrValues extendedModules;})
       rawNixosConfigs;
 
     eachDefaultSystemOutputs = flake-utils.lib.eachDefaultSystem (system: let
       pkgs = importNixpkgs system [rust-overlay.overlays.default];
+
       treefmtEval = loadTreefmt pkgs;
-      toplevel = name: config: nameValuePair "nixosConfigs/${name}" config.config.system.build.toplevel;
+      toplevel = name: config: nameValuePair "nixosConfigurations/${name}" config.config.system.build.toplevel;
     in {
       packages = importPackages pkgs;
+
       formatter = treefmtEval.config.build.wrapper;
       checks = mapAttrs' toplevel nixosConfigurations;
     });
@@ -143,7 +174,10 @@
         nonBrokenPkgs;
       in {
         packages.${system} = nonBrokenPkgs;
-        tests.${system} = passthruTests;
+        tests.${system} = {
+          passthru = passthruTests;
+          nixos = pickNixosTests (importProjects {pkgs = pkgs // nonBrokenPkgs;});
+        };
 
         nixosConfigurations.${system} =
           mapAttrs
@@ -157,14 +191,14 @@
 
       nixosModules =
         (import ./modules/all-modules.nix)
+        // (flattenAttrsSlash (pickNixosModules (importProjects {})))
         // {
           # The default module adds the default overlay on top of nixpkgs.
           # This is so that `ngipkgs` can be used alongside `nixpkgs` in a configuration.
           default.nixpkgs.overlays = [self.overlays.default];
         };
 
-      # Overlays a package set (e.g. nixpkgs) with the packages defined in this flake.
-      overlays.default = final: prev: importPackages prev;
+      overlays.default = overlay;
     };
   in
     foldr recursiveUpdate {} [

modules/all-modules.nix

@@ -1,10 +1,3 @@
 {
-  # LiberaForms is intentionally disabled.
-  # Refer to <https://github.com/ngi-nix/ngipkgs/issues/40>.
-  #liberaforms = import ./liberaforms.nix;
-  flarum = import ./flarum.nix;
-  kbin = import ./kbin.nix;
-  mcaptcha = import ./mcaptcha.nix;
-  pretalx = import ./pretalx.nix;
   unbootable = import ./unbootable.nix;
 }

pkgs/by-name/default.nix

@@ -25,18 +25,18 @@
 
   packageDirectories = concatMapAttrs names (readDir baseDirectory);
 
-  callModule = moduleDir: let
+  callModule = module: let
     evaluated = lib.evalModules {
       specialArgs = {
         inherit dream2nix;
         packageSets.nixpkgs = pkgs;
       };
       modules = [
-        moduleDir
+        module
         {
           paths.projectRoot = ../..;
           paths.projectRootFile = "flake.nix";
-          paths.package = moduleDir;
+          paths.package = module;
           paths.lockFile = "lock.json";
         }
       ];

pkgs/by-name/flarum/package.nix

@@ -1,33 +1,27 @@
 {
   fetchFromGitHub,
-  fetchurl,
   lib,
   php,
-}: let
-  inherit
-    (lib)
-    licenses
-    ;
-in
-  php.buildComposerProject (finalAttrs: {
-    pname = "flarum";
-    version = "1.8.0";
+}:
+php.buildComposerProject (finalAttrs: {
+  pname = "flarum";
+  version = "1.8.0";
 
-    src = fetchFromGitHub {
-      owner = "flarum";
-      repo = "flarum";
-      rev = "v${finalAttrs.version}";
-      hash = "sha256-xadZIdyH20mxfxCyiDRtSRSrPj8DWXpuup61WSsjgWw=";
-    };
+  src = fetchFromGitHub {
+    owner = "flarum";
+    repo = "flarum";
+    rev = "v${finalAttrs.version}";
+    hash = "sha256-xadZIdyH20mxfxCyiDRtSRSrPj8DWXpuup61WSsjgWw=";
+  };
 
-    composerLock = ./composer.lock;
-    composerStrictValidation = false;
-    vendorHash = "sha256-G/EPHcvcppuyAC0MAzE11ZjlOSTlphQrHnO3yS4+j5g=";
+  composerLock = ./composer.lock;
+  composerStrictValidation = false;
+  vendorHash = "sha256-G/EPHcvcppuyAC0MAzE11ZjlOSTlphQrHnO3yS4+j5g=";
 
-    meta = {
-      changelog = "https://github.com/flarum/framework/blob/main/CHANGELOG.md";
-      description = "Flarum is a delightfully simple discussion platform for your website";
-      homepage = "https://github.com/flarum/flarum";
-      license = licenses.mit;
-    };
-  })
+  meta = {
+    changelog = "https://github.com/flarum/framework/blob/main/CHANGELOG.md";
+    description = "Flarum is a delightfully simple discussion platform for your website";
+    homepage = "https://github.com/flarum/flarum";
+    license = lib.licenses.mit;
+  };
+})

pkgs/by-name/kbin/package.nix

@@ -1,4 +1,5 @@
 {
+  lib,
   runCommand,
   kbin-frontend,
   kbin-backend,
@@ -10,9 +11,14 @@ runCommand "kbin" {
     kbin-backend.passthru
     // {
       tests = {
-        inherit (nixosTests) kbin;
+        inherit (nixosTests.Kbin) kbin;
       };
     };
+  meta = {
+    license = lib.licenses.agpl3Only;
+    homepage = "https://kbin.pub/";
+    description = "/kbin is a modular, decentralized content aggregator and microblogging platform running on the Fediverse network.";
+  };
 } ''
   # As of 2023-10-09, there is no way to just symlink
   # backend and frontend (using `lndir`):

pkgs/by-name/mcaptcha-cache/package.nix

@@ -1,7 +1,6 @@
 {
   rustPlatform,
   fetchFromGitHub,
-  lib,
 }: let
   src = fetchFromGitHub {
     owner = "mCaptcha";
@@ -13,7 +12,7 @@
     sha256 = "sha256-whRLgYkoBoVQiZwrmwBwqgHzPqqXC6g3na3YrH4/xVo=";
   };
 in
-  rustPlatform.buildRustPackage rec {
+  rustPlatform.buildRustPackage {
     inherit src;
     pname = "cache";
     version = "unstable-2023-03-08";

pkgs/by-name/mcaptcha/package.nix

@@ -197,7 +197,7 @@
         mainProgram = "mcaptcha";
       };
 
-      passthru.tests = {inherit (nixosTests) mcaptcha-create-locally mcaptcha-bring-your-own-services;};
+      passthru.tests = {inherit (nixosTests.mCaptcha) create-locally bring-your-own-services;};
     };
 in
   mcaptcha

pkgs/by-name/rosenpass/package.nix

@@ -48,7 +48,7 @@ in
       installManPage doc/rosenpass.1
     '';
 
-    passthru.tests.rosenpass = nixosTests.rosenpass;
+    passthru.tests.rosenpass = nixosTests.Rosenpass.rosenpass;
 
     meta = {
       description = "Build post-quantum-secure VPNs with WireGuard!";