commit b9c0d93670275e69df24902b05bf4aa4f0fcbe96
parent 11287a8436e0700b65d3a3edef48f9344500bbd6
Author: Daniel Barlow <dan@telent.net>
Date: Sun, 11 Feb 2024 23:47:11 +0000
build modules at same time as main kernel vmlinux
This changes the practice for building kernel modules: now we expect
that the appropriate Kconfig symbols are set to =m in
config.kernel.config, and then use pkgs.kmodloader to create
a service that loads and unloads all the modules depended on by
a particular requirement.
Note that modules won't be installed on the target device just by
virue of having been built: only the modules that are referenced by a
kmodloader package will be in the closure.
An example may make this clearer: see modules/firewall/default.nix
in this commit.
Why?
If you have a compiled Linux kernel source tree and you change some
symbol from "is not set" to m and then run make modules, you cannot in
general expect that newly compiled module to work. This is because
there are places in the build of the main kernel where it looks to see
which modules _may_ be defined and uses that information to
accommodate them.
For example in an in-kernel build of
https://github.com/torvalds/linux/blob/master/net/netfilter/core.c#L689
some symbols are defined only if CONFIG_NF_CONNTRACK is set, meaning
this code won't work if we have it unset initially then try later to
enable it and build modules only. Or see
https://github.com/torvalds/linux/blob/master/include/linux/netdevice.h#L160
Diffstat:
7 files changed, 78 insertions(+), 104 deletions(-)
diff --git a/modules/firewall/default.nix b/modules/firewall/default.nix
@@ -10,45 +10,8 @@ let
inherit (pkgs) liminix;
inherit (pkgs.liminix.services) oneshot;
- kconf = isModule :
- # setting isModule false is utterly untested and mostly
- # unimplemented: I say this to preempt any "how on earth is this
- # even supposed to work?" questions
- let yes = if isModule then "m" else "y";
- in {
- NETFILTER = "y";
- NETFILTER_ADVANCED = "y";
- NETFILTER_NETLINK = yes;
- NF_CONNTRACK = yes;
-
- IP6_NF_IPTABLES= yes;
- IP_NF_IPTABLES = yes;
- IP_NF_NAT = yes;
- IP_NF_TARGET_MASQUERADE = yes;
-
- NFT_CT = yes;
- NFT_FIB_IPV4 = yes;
- NFT_FIB_IPV6 = yes;
- NFT_LOG = yes;
- NFT_MASQ = yes;
- NFT_NAT = yes;
- NFT_REJECT = yes;
- NFT_REJECT_INET = yes;
-
- NF_CT_PROTO_DCCP = "y";
- NF_CT_PROTO_SCTP = "y";
- NF_CT_PROTO_UDPLITE = "y";
- NF_LOG_SYSLOG = yes;
- NF_NAT = yes;
- NF_NAT_MASQUERADE = "y";
- NF_TABLES = yes;
- NF_TABLES_INET = "y";
- NF_TABLES_IPV4 = "y";
- NF_TABLES_IPV6 = "y";
- };
- kmodules = pkgs.kernel-modules.override {
- kernelSrc = config.system.outputs.kernel.src;
- modulesupport = config.system.outputs.kernel.modulesupport;
+ kmodules = pkgs.kmodloader.override {
+ inherit (config.system.outputs) kernel;
targets = [
"nft_fib_ipv4"
"nft_fib_ipv6"
@@ -82,12 +45,6 @@ let
"xt_nat"
"xt_tcpudp"
];
- kconfig = kconf true;
- };
- loadModules = oneshot {
- name = "firewall-modules";
- up = "sh ${kmodules}/load.sh";
- down = "sh ${kmodules}/unload.sh";
};
in
{
@@ -107,11 +64,41 @@ in
in svc // {
build = args :
let args' = args // {
- dependencies = (args.dependencies or []) ++ [loadModules];
+ dependencies = (args.dependencies or []) ++ [kmodules];
};
in svc.build args' ;
};
- kernel.config = kconf true;
+ kernel.config = {
+ NETFILTER = "y";
+ NETFILTER_ADVANCED = "y";
+ NETFILTER_NETLINK = "m";
+ NF_CONNTRACK = "m";
+
+ IP6_NF_IPTABLES= "m";
+ IP_NF_IPTABLES = "m";
+ IP_NF_NAT = "m";
+ IP_NF_TARGET_MASQUERADE = "m";
+
+ NFT_CT = "m";
+ NFT_FIB_IPV4 = "m";
+ NFT_FIB_IPV6 = "m";
+ NFT_LOG = "m";
+ NFT_MASQ = "m";
+ NFT_NAT = "m";
+ NFT_REJECT = "m";
+ NFT_REJECT_INET = "m";
+
+ NF_CT_PROTO_DCCP = "y";
+ NF_CT_PROTO_SCTP = "y";
+ NF_CT_PROTO_UDPLITE = "y";
+ NF_LOG_SYSLOG = "m";
+ NF_NAT = "m";
+ NF_NAT_MASQUERADE = "y";
+ NF_TABLES = "m";
+ NF_TABLES_INET = "y";
+ NF_TABLES_IPV4 = "y";
+ NF_TABLES_IPV6 = "y";
+ };
};
}
diff --git a/modules/kernel/default.nix b/modules/kernel/default.nix
@@ -80,8 +80,7 @@ in {
config.kernel.conditionalConfig;
k = liminix.builders.kernel.override {
config = mergedConfig;
- version = builtins.trace config.kernel.version config.kernel.version;
- inherit (config.kernel) src extraPatchPhase;
+ inherit (config.kernel) version src extraPatchPhase;
targets = config.kernel.makeTargets;
};
in {
diff --git a/pkgs/default.nix b/pkgs/default.nix
@@ -67,6 +67,7 @@ in {
initramfs-peek = callPackage ./initramfs-peek {};
kernel-backport = callPackage ./kernel-backport {};
kernel-modules = callPackage ./kernel-modules {};
+ kmodloader = callPackage ./kmodloader {};
levitate = callPackage ./levitate {};
libubootenv = callPackage ./libubootenv {};
linotify = callPackage ./linotify {};
diff --git a/pkgs/kernel-modules/Makefile b/pkgs/kernel-modules/Makefile
@@ -1,3 +0,0 @@
-
-
-# obj-m += net/ipv4/netfilter/nft_fib_ipv4.o
diff --git a/pkgs/kernel-modules/default.nix b/pkgs/kernel-modules/default.nix
@@ -1,50 +0,0 @@
-{
- stdenv
-, buildPackages
-, kernelSrc ? null
-, modulesupport ? null
-, targets ? []
-, kconfig ? {}
-, openssl
-, writeText
-, lib
-}:
-let
- writeConfig = import ../kernel/write-kconfig.nix { inherit lib writeText; };
- arch = stdenv.hostPlatform.linuxArch;
-in stdenv.mkDerivation {
- name = "kernel-modules";
-
- nativeBuildInputs = [buildPackages.stdenv.cc] ++
- (with buildPackages.pkgs; [
- bc bison flex
- openssl
- cpio
- kmod
- ]);
- CC = "${stdenv.cc.bintools.targetPrefix}gcc";
- HOST_EXTRACFLAGS = with buildPackages.pkgs;
- "-I${buildPackages.openssl.dev}/include -L${buildPackages.openssl.out}/lib";
- CROSS_COMPILE = stdenv.cc.bintools.targetPrefix;
- ARCH = arch;
- KBUILD_BUILD_HOST = "liminix.builder";
-
- buildPhase = ''
- cat ${writeConfig "kconfig" kconfig} > .more-config
- cat .more-config >> .config
- make olddefconfig
- for v in $(cat .more-config) ; do grep $v .config || (echo Missing $v && exit 1);done
- make modules
- '';
- src = modulesupport;
- installPhase = ''
- mkdir -p $out/lib/modules/0.0
- find . -name \*.ko | cpio --verbose --make-directories -p $out/lib/modules/0.0
- depmod -b $out -v 0.0
- touch $out/load.sh
- for i in ${lib.concatStringsSep " " targets}; do
- modprobe -S 0.0 -d $out --show-depends $i >> $out/load.sh
- done
- tac < $out/load.sh | sed 's/^insmod/rmmod/g' > $out/unload.sh
- '';
-}
diff --git a/pkgs/kernel/default.nix b/pkgs/kernel/default.nix
@@ -104,8 +104,7 @@ stdenv.mkDerivation rec {
mkdir -p $headers
cp -a include .config $headers/
mkdir -p $modulesupport
- cp modules.* $modulesupport
- make clean modules_prepare
+ make modules
cp -a . $modulesupport
'';
}
diff --git a/pkgs/kmodloader/default.nix b/pkgs/kmodloader/default.nix
@@ -0,0 +1,41 @@
+{
+ liminix
+, lib
+, targets ? []
+, kernel ? null
+, runCommand
+, pkgsBuildBuild
+} :
+let
+ inherit (liminix.services) oneshot;
+ inherit (lib) concatStringsSep;
+ loader = runCommand "modules" {
+ nativeBuildInputs = with pkgsBuildBuild ;[
+ kmod cpio gawk
+ ];
+ } ''
+ kernel=${kernel.modulesupport}
+
+ mkdir -p lib/modules/0.0
+ (cd $kernel && find . -name \*.ko | cpio --verbose --make-directories -p $NIX_BUILD_TOP/lib/modules/0.0)
+ cp $kernel/modules.* lib/modules/0.0
+ depmod -b . 0.0
+
+ (for i in ${lib.concatStringsSep " " targets}; do
+ modprobe -S 0.0 -d $NIX_BUILD_TOP --show-depends $i | sed "s,^insmod $NIX_BUILD_TOP/lib/modules/0.0/,,g"
+ done) | awk '!a[$0]++' > load-order
+
+ mkdir $out
+ for i in $(cat load-order); do
+ install -v $NIX_BUILD_TOP/lib/modules/0.0/$i -D $out/$i
+ done
+ echo "O=$out" > $out/load.sh
+ sed "s,^,insmod \$O/,g" < load-order >> $out/load.sh
+ echo "O=$out" > $out/unload.sh
+ tac load-order | sed "s,^,rmmod \$O/,g" > $out/unload.sh
+ '';
+in oneshot {
+ name = "kmodloader-" + (concatStringsSep "-" targets);
+ up = "sh ${loader}/load.sh";
+ down = "sh ${loader}/unload.sh";
+}
