From b9c0d93670275e69df24902b05bf4aa4f0fcbe96 Mon Sep 17 00:00:00 2001 From: Daniel Barlow Date: Sun, 11 Feb 2024 23:47:11 +0000 Subject: [PATCH] build modules at same time as main kernel vmlinux This changes the practice for building kernel modules: now we expect that the appropriate Kconfig symbols are set to =m in config.kernel.config, and then use pkgs.kmodloader to create a service that loads and unloads all the modules depended on by a particular requirement. Note that modules won't be installed on the target device just by virue of having been built: only the modules that are referenced by a kmodloader package will be in the closure. An example may make this clearer: see modules/firewall/default.nix in this commit. Why? If you have a compiled Linux kernel source tree and you change some symbol from "is not set" to m and then run make modules, you cannot in general expect that newly compiled module to work. This is because there are places in the build of the main kernel where it looks to see which modules _may_ be defined and uses that information to accommodate them. For example in an in-kernel build of https://github.com/torvalds/linux/blob/master/net/netfilter/core.c#L689 some symbols are defined only if CONFIG_NF_CONNTRACK is set, meaning this code won't work if we have it unset initially then try later to enable it and build modules only. Or see https://github.com/torvalds/linux/blob/master/include/linux/netdevice.h#L160 --- modules/firewall/default.nix | 81 ++++++++++++++------------------- modules/kernel/default.nix | 3 +- pkgs/default.nix | 1 + pkgs/kernel-modules/Makefile | 3 -- pkgs/kernel-modules/default.nix | 50 -------------------- pkgs/kernel/default.nix | 3 +- pkgs/kmodloader/default.nix | 41 +++++++++++++++++ 7 files changed, 78 insertions(+), 104 deletions(-) delete mode 100644 pkgs/kernel-modules/Makefile delete mode 100644 pkgs/kernel-modules/default.nix create mode 100644 pkgs/kmodloader/default.nix diff --git a/modules/firewall/default.nix b/modules/firewall/default.nix index b8ec08f..1aa4769 100644 --- a/modules/firewall/default.nix +++ b/modules/firewall/default.nix @@ -10,45 +10,8 @@ let inherit (pkgs) liminix; inherit (pkgs.liminix.services) oneshot; - kconf = isModule : - # setting isModule false is utterly untested and mostly - # unimplemented: I say this to preempt any "how on earth is this - # even supposed to work?" questions - let yes = if isModule then "m" else "y"; - in { - NETFILTER = "y"; - NETFILTER_ADVANCED = "y"; - NETFILTER_NETLINK = yes; - NF_CONNTRACK = yes; - - IP6_NF_IPTABLES= yes; - IP_NF_IPTABLES = yes; - IP_NF_NAT = yes; - IP_NF_TARGET_MASQUERADE = yes; - - NFT_CT = yes; - NFT_FIB_IPV4 = yes; - NFT_FIB_IPV6 = yes; - NFT_LOG = yes; - NFT_MASQ = yes; - NFT_NAT = yes; - NFT_REJECT = yes; - NFT_REJECT_INET = yes; - - NF_CT_PROTO_DCCP = "y"; - NF_CT_PROTO_SCTP = "y"; - NF_CT_PROTO_UDPLITE = "y"; - NF_LOG_SYSLOG = yes; - NF_NAT = yes; - NF_NAT_MASQUERADE = "y"; - NF_TABLES = yes; - NF_TABLES_INET = "y"; - NF_TABLES_IPV4 = "y"; - NF_TABLES_IPV6 = "y"; - }; - kmodules = pkgs.kernel-modules.override { - kernelSrc = config.system.outputs.kernel.src; - modulesupport = config.system.outputs.kernel.modulesupport; + kmodules = pkgs.kmodloader.override { + inherit (config.system.outputs) kernel; targets = [ "nft_fib_ipv4" "nft_fib_ipv6" @@ -82,12 +45,6 @@ let "xt_nat" "xt_tcpudp" ]; - kconfig = kconf true; - }; - loadModules = oneshot { - name = "firewall-modules"; - up = "sh ${kmodules}/load.sh"; - down = "sh ${kmodules}/unload.sh"; }; in { @@ -107,11 +64,41 @@ in in svc // { build = args : let args' = args // { - dependencies = (args.dependencies or []) ++ [loadModules]; + dependencies = (args.dependencies or []) ++ [kmodules]; }; in svc.build args' ; }; - kernel.config = kconf true; + kernel.config = { + NETFILTER = "y"; + NETFILTER_ADVANCED = "y"; + NETFILTER_NETLINK = "m"; + NF_CONNTRACK = "m"; + + IP6_NF_IPTABLES= "m"; + IP_NF_IPTABLES = "m"; + IP_NF_NAT = "m"; + IP_NF_TARGET_MASQUERADE = "m"; + + NFT_CT = "m"; + NFT_FIB_IPV4 = "m"; + NFT_FIB_IPV6 = "m"; + NFT_LOG = "m"; + NFT_MASQ = "m"; + NFT_NAT = "m"; + NFT_REJECT = "m"; + NFT_REJECT_INET = "m"; + + NF_CT_PROTO_DCCP = "y"; + NF_CT_PROTO_SCTP = "y"; + NF_CT_PROTO_UDPLITE = "y"; + NF_LOG_SYSLOG = "m"; + NF_NAT = "m"; + NF_NAT_MASQUERADE = "y"; + NF_TABLES = "m"; + NF_TABLES_INET = "y"; + NF_TABLES_IPV4 = "y"; + NF_TABLES_IPV6 = "y"; + }; }; } diff --git a/modules/kernel/default.nix b/modules/kernel/default.nix index 8455daf..5dcc292 100644 --- a/modules/kernel/default.nix +++ b/modules/kernel/default.nix @@ -80,8 +80,7 @@ in { config.kernel.conditionalConfig; k = liminix.builders.kernel.override { config = mergedConfig; - version = builtins.trace config.kernel.version config.kernel.version; - inherit (config.kernel) src extraPatchPhase; + inherit (config.kernel) version src extraPatchPhase; targets = config.kernel.makeTargets; }; in { diff --git a/pkgs/default.nix b/pkgs/default.nix index 97d15b9..d628a08 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -67,6 +67,7 @@ in { initramfs-peek = callPackage ./initramfs-peek {}; kernel-backport = callPackage ./kernel-backport {}; kernel-modules = callPackage ./kernel-modules {}; + kmodloader = callPackage ./kmodloader {}; levitate = callPackage ./levitate {}; libubootenv = callPackage ./libubootenv {}; linotify = callPackage ./linotify {}; diff --git a/pkgs/kernel-modules/Makefile b/pkgs/kernel-modules/Makefile deleted file mode 100644 index ac6930d..0000000 --- a/pkgs/kernel-modules/Makefile +++ /dev/null @@ -1,3 +0,0 @@ - - -# obj-m += net/ipv4/netfilter/nft_fib_ipv4.o diff --git a/pkgs/kernel-modules/default.nix b/pkgs/kernel-modules/default.nix deleted file mode 100644 index 917fce9..0000000 --- a/pkgs/kernel-modules/default.nix +++ /dev/null @@ -1,50 +0,0 @@ -{ - stdenv -, buildPackages -, kernelSrc ? null -, modulesupport ? null -, targets ? [] -, kconfig ? {} -, openssl -, writeText -, lib -}: -let - writeConfig = import ../kernel/write-kconfig.nix { inherit lib writeText; }; - arch = stdenv.hostPlatform.linuxArch; -in stdenv.mkDerivation { - name = "kernel-modules"; - - nativeBuildInputs = [buildPackages.stdenv.cc] ++ - (with buildPackages.pkgs; [ - bc bison flex - openssl - cpio - kmod - ]); - CC = "${stdenv.cc.bintools.targetPrefix}gcc"; - HOST_EXTRACFLAGS = with buildPackages.pkgs; - "-I${buildPackages.openssl.dev}/include -L${buildPackages.openssl.out}/lib"; - CROSS_COMPILE = stdenv.cc.bintools.targetPrefix; - ARCH = arch; - KBUILD_BUILD_HOST = "liminix.builder"; - - buildPhase = '' - cat ${writeConfig "kconfig" kconfig} > .more-config - cat .more-config >> .config - make olddefconfig - for v in $(cat .more-config) ; do grep $v .config || (echo Missing $v && exit 1);done - make modules - ''; - src = modulesupport; - installPhase = '' - mkdir -p $out/lib/modules/0.0 - find . -name \*.ko | cpio --verbose --make-directories -p $out/lib/modules/0.0 - depmod -b $out -v 0.0 - touch $out/load.sh - for i in ${lib.concatStringsSep " " targets}; do - modprobe -S 0.0 -d $out --show-depends $i >> $out/load.sh - done - tac < $out/load.sh | sed 's/^insmod/rmmod/g' > $out/unload.sh - ''; -} diff --git a/pkgs/kernel/default.nix b/pkgs/kernel/default.nix index 0a1b065..4ddf1dd 100644 --- a/pkgs/kernel/default.nix +++ b/pkgs/kernel/default.nix @@ -104,8 +104,7 @@ stdenv.mkDerivation rec { mkdir -p $headers cp -a include .config $headers/ mkdir -p $modulesupport - cp modules.* $modulesupport - make clean modules_prepare + make modules cp -a . $modulesupport ''; } diff --git a/pkgs/kmodloader/default.nix b/pkgs/kmodloader/default.nix new file mode 100644 index 0000000..7dbfe4a --- /dev/null +++ b/pkgs/kmodloader/default.nix @@ -0,0 +1,41 @@ +{ + liminix +, lib +, targets ? [] +, kernel ? null +, runCommand +, pkgsBuildBuild +} : +let + inherit (liminix.services) oneshot; + inherit (lib) concatStringsSep; + loader = runCommand "modules" { + nativeBuildInputs = with pkgsBuildBuild ;[ + kmod cpio gawk + ]; + } '' + kernel=${kernel.modulesupport} + + mkdir -p lib/modules/0.0 + (cd $kernel && find . -name \*.ko | cpio --verbose --make-directories -p $NIX_BUILD_TOP/lib/modules/0.0) + cp $kernel/modules.* lib/modules/0.0 + depmod -b . 0.0 + + (for i in ${lib.concatStringsSep " " targets}; do + modprobe -S 0.0 -d $NIX_BUILD_TOP --show-depends $i | sed "s,^insmod $NIX_BUILD_TOP/lib/modules/0.0/,,g" + done) | awk '!a[$0]++' > load-order + + mkdir $out + for i in $(cat load-order); do + install -v $NIX_BUILD_TOP/lib/modules/0.0/$i -D $out/$i + done + echo "O=$out" > $out/load.sh + sed "s,^,insmod \$O/,g" < load-order >> $out/load.sh + echo "O=$out" > $out/unload.sh + tac load-order | sed "s,^,rmmod \$O/,g" > $out/unload.sh + ''; +in oneshot { + name = "kmodloader-" + (concatStringsSep "-" targets); + up = "sh ${loader}/load.sh"; + down = "sh ${loader}/unload.sh"; +}