extract log persistence config from s6 to new module

because it frobs kernel config, it breaks levitate
as levitate evalModules doesn't include the kernel

THOUGHTS.txt

@@ -6670,3 +6670,128 @@ Mon Dec 23 18:28:50 GMT 2024
 
 it might be worth moving ubi option decls into the hardware module, if
 they're hardware-dependent
+
+Tue Dec 24 16:14:50 GMT 2024
+
+Where next?
+
+a config for haproxy would be good
+
+> a connection or request arrives on a frontend, then the
+information carried with this request or connection are processed, and
+at this point it is possible to write ACLs-based conditions making use
+of these information to decide what backend will process the request.
+
+http://docs.haproxy.org/3.1/intro.html#3.4.4
+
+listen foo
+	bind :443
+
+frontend foo
+	mode tcp
+
+Thu Dec 26 14:27:26 GMT 2024
+
+What's the plan?
+
+1) build the updater target for rotuer
+2) ssh forward through bordervm to install it
+3) carve the ngix sni proxy into examples/module-sniproxy
+  with a fat warning comment
+3.5) should  we add the examples to ci?
+4) see if it builds
+5) using curl on bordervm, see if it forwards
+5000) swap the real rotuer hardware
+
+
+Sun Dec 29 18:22:42 GMT 2024
+
+To make sniproxy work it needs dns, which means it needs an upstream.
+But if we move the bordervm cable from lan to wan, we won't be able to
+rebuild over ssh unless it's sufficiently unbroken for pppoe to be working
+
+... unless we add a "management" address to the wan interface along with pppoe
+
+to permit wan stuff through the firewall, now that we've fixed the
+firewall :embarrassed:, do
+
+/nix/store/*nftables*/bin/nft insert rule table-ip input-ip4 position 19  iifname "wan" jump incoming-allowed-ip4
+
+What else do we need to try rt3200 as prod rotuer?
+
+- pstore logging
+- log shipping (maybe via wan to bordervm for now)
+- enable sniproxy module
+
+zcat /proc/config.gz  | grep PSTORE
+
+Tue Dec 31 22:45:33 GMT 2024
+
+/nix/store/i1khbsqpyx020xrhvfbdazc1bnmirc72-kernel-aarch64-unknown-linux-musl-modulesupport/vmlinux
+
+
+these 3 derivations will be built:
+  /nix/store/fvxrvyx64cm86cc4na0584qj9xw6s406-kernel-aarch64-unknown-linux-musl.drv
+  /nix/store/ksg93bd8x2z1xpfmj24ixm2srg547mjx-dtb-aarch64-unknown-linux-musl.drv
+  /nix/store/n33ij1npzwjmlsw23sz6yhyrh9hgxwaw-kernel.image-aarch64-unknown-linux-musl.drv
+building '/nix/store/fvxrvyx64cm86cc4na0584qj9xw6s406-kernel-aarch64-unknown-linux-musl.drv'...
+zcat /proc/config.gz  | grep PSTORE
+
+we think tftpboot works (check!)
+build the FIT from the updater target with squashfs added in rotuer.nix
+boot it with tftp by copy-pasting the boot.scr and changing it
+
+setenv serverip 10.0.0.1
+setenv ipaddr 10.0.0.8
+tftpboot 0x4007ff28 r2/rootfs; tftpboot 0x41086f28 r2/image; tftpboot 0x4107ff28 r2/dtb
+ bootm 0x41086f28 - 0x4107ff28
+
+tftpboot works
+tftpboot with outputs.uimage works
+1458ed2cdeaa6bf4c02c6511a6d7369d  /nix/store/4n3jc4n7lsp73yc2ccnmgibc7079rxms-kernel.image-aarch64-unknown-linux-musl
+
+$ grep fit /nix/store/ihi7vski37b9ph4xcyqpabymc5nsngjd-system-configuration-aarch64-unknown-linux-musl/etc/nix-store-paths
+/nix/store/5jphs9mnb8hzhvwfi2z8h6cnaz6qx4y6-boot-fit
+$ md5sum /nix/store/5jphs9mnb8hzhvwfi2z8h6cnaz6qx4y6-boot-fit/fit
+1458ed2cdeaa6bf4c02c6511a6d7369d  /nix/store/5jphs9mnb8hzhvwfi2z8h6cnaz6qx4y6-boot-fit/fit
+
+something is changing /persist/boot from a symlink to a directory and I don't know what
+... plot twist: or is it? maybe it's ubimage that makes it a directory
+
+rt3200 has no pmsg-size according to  dtc -I fs  /proc/device-tree/ -O dts
+
+should be more like
+
+reserved-memory {
+		ramoops@03f00000 {
+			compatible = "ramoops";
+			reg = <0x03f00000 0x10000>;
+			pmsg-size = <0x10000>;
+		};
+	};
+};
+
+Wed Jan  1 14:57:35 GMT 2025
+
+At last I have working persistent logging.
+
+I think we need to do something at boot time to move the persistent logs into
+the regular s6 log thing
+
+      foreground {
+        redirfd -w 1 /run/prior-boot2
+        elglob file /sys/fs/pstore/* cat $file
+      }
+
+Thu Jan  2 14:31:11 GMT 2025
+
+to change to a previous "generation" we could run any of
+"/nix/store/*system-configuration*/bin/install /mnt" from a rescue
+system. It would populate /boot and bin/activate
+
+supposing we are in such a rescue system, how do we find *which*
+system-configuration is the one we want to revert to? The derivation
+should be pure, so if we're going to timestamp anything we have to do
+that in the imperative step i.e. update.sh
+
+perhaps a symlink from /persist/configuration/yyyymmddtmmhhss -> /nix/store/eeee-blah

default.nix

@@ -33,6 +33,7 @@ let
       ./modules/busybox.nix
       ./modules/hostname.nix
       ./modules/kernel
+      ./modules/logging.nix
       ./modules/klogd.nix
       device.module
       liminix-config

devices/turris-omnia/default.nix

@@ -173,8 +173,13 @@
         ../../modules/outputs/tftpboot.nix
         ../../modules/outputs/mbrimage.nix
       ];
+    };
+
 
       config = {
+        rootfsType = lib.mkDefault "btrfs"; # override this if you are building tftpboot
+        rootOptions = lib.mkDefault "subvol=@";
+
         services.mtd-name-links = mtd_by_name_links;
         kernel = {
           src = pkgs.pkgsBuildBuild.fetchurl {
@@ -292,6 +297,7 @@
           };
         };
         boot = {
+          loader.extlinux.enable = lib.mkDefault true; # override this if you are building tftpboot
           commandLine = [
             "console=ttyS0,115200"
             "pcie_aspm=off" # ath9k pci incompatible with PCIe ASPM

doc/admin.rst

@@ -188,16 +188,54 @@ the current system closure. Note that Liminix does not have the NixOS
 concept of environments or generations, and there is no way back from
 this except for building the previous configuration again.
 
-
 Caveats
 -------
 
 * it needs there to be enough free space on the device for all the new
-  packages in addition to all the packages already on it - which may be
-  a problem if a lot of things have changed (e.g. a new version of
-  nixpkgs).
+  packages in addition to all the packages already on it - which may
+  be a problem if there is little flash storage or if a lot of things
+  have changed (e.g. a new version of nixpkgs).
+
+* it may not be able to upgrade the kernel: this is device-dependent.
+  If your device boots from a kernel image on a raw MTD partition or
+  or UBI volume, update.sh is unable to alter the kernel partition.
+  If your device boots from a kernel inside the filesystem (e.g. using
+  bootloader.extlinux or bootloder.fit) then the kernel will be
+  upgraded along with the userland
+
+
+Recovery/downgrades
+-------------------
+
+The :command:`update.sh` script also creates a timestamped symlink on
+the device which points to the system configuration it installs. If
+you install a configuration that doesn't work, you can revert to any
+other installed configuration by
+
+1) booting to some kind of rescue or recovery system (which may be
+   some vendor-provided rescue option, or your own recovery system
+   perhaps based on :file:`examples/recovery.nix`) and mounting
+   your Liminix filesystem on /mnt
+
+2) picking another previously-installed configuration that _did_ work,
+   and switching back to it:
+
+.. code-block:: console
+   
+    # ls  -ld /mnt/*configuration
+    lrwxrwxrwx    1        90 /mnt/20252102T182104.configuration -> nix/store/v1w0h4zw65ah4c2r0k7nyy125qrxhq78-system-configuration-aarch64-unknown-linux-musl
+    lrwxrwxrwx    1        90 /mnt/20251802T181822.configuration -> nix/store/wqjl9s9xljl2wg8257292zghws9ssidk-system-configuration-aarch64-unknown-linux-musl
+    # : 20251802T181822 is the working system, so reinstall it
+    # /mnt/20251802T181822.configuration/bin/install /mnt
+    # umount /mnt
+    # reboot
+    
+This will install the previous configuration's activation binary into
+/bin, and copy its kernel and initramfs into /boot. Note that it
+depends on the previous system not having been garbage-collected.
+
+
 
-* it cannot upgrade the kernel, only userland
 
 .. _levitate:  
 

modules/logging.nix

@@ -0,0 +1,19 @@
+{ config, lib, ... }:
+let
+  inherit (lib) mkIf mkEnableOption mkOption types;
+in {
+  options = {
+    logging = {
+      persistent = {
+        enable = mkEnableOption "store logs across reboots";
+      };
+    };
+  };
+   config = {
+     kernel.config = mkIf config.logging.persistent.enable {
+       PSTORE = "y";
+       PSTORE_PMSG = "y";
+       PSTORE_RAM = "y";
+     };
+   };
+}

modules/outputs.nix

@@ -19,6 +19,7 @@ in
     ./outputs/uimage.nix
     ./outputs/updater
     ./outputs/ubimage.nix
+    ./outputs/mtdimage.nix
   ];
   options = {
     system.outputs = {

modules/outputs/updater/update.sh

@@ -37,8 +37,8 @@ echo installing from systemConfiguration $toplevel to host $target_host
 
 $ssh_command $target_host uname -a || die "Can't ssh to $target_host"
 min-copy-closure $target_host $toplevel
-set -x
-$ssh_command $target_host $toplevel/bin/install
+ts=$(date +%Y%m%dT%H%M%S)
+$ssh_command $target_host "$toplevel/bin/install && ln -s $(realpath --relative-to / $toplevel) /persist/${ts}.configuration"
 case "$reboot" in
     reboot)
 	$ssh_command $target_host "sync; source /etc/profile; reboot"

modules/s6/default.nix

@@ -17,7 +17,7 @@ let
   logger =
     let pipecmds =
           ["${s6}/bin/s6-log -bpd3 -- ${cfg.script} 1"] ++
-          (lib.optional cfg.persistent.enable
+          (lib.optional (cfg ? persistent && cfg.persistent.enable)
             "/bin/tee /dev/pmsg0") ++
           (lib.optional cfg.shipping.enable
             "${pkgs.logshipper}/bin/logtap ${cfg.shipping.socket} logshipper-socket-event");
@@ -215,9 +215,6 @@ let
 in {
   options = {
     logging = {
-      persistent = {
-        enable = mkEnableOption "store logs across reboots";
-      };
       shipping = {
         enable = mkEnableOption "unix socket for log shipping";
         socket = mkOption {
@@ -269,11 +266,6 @@ in {
     )];
 
   config = {
-    kernel.config = mkIf config.logging.persistent.enable {
-      PSTORE = "y";
-      PSTORE_PMSG = "y";
-      PSTORE_RAM = "y";
-    };
     filesystem = dir {
       etc = dir {
         s6-rc = dir {

pkgs/logshipper/logtap.c

@@ -35,24 +35,24 @@ int open_shipper_socket(char *pathname) {
     static int fail_count = 0;
 
     struct sockaddr_un sa = {
-	.sun_family = AF_LOCAL
+        .sun_family = AF_LOCAL
     };
     strncpy(sa.sun_path, pathname, sizeof(sa.sun_path) - 1);
 
     fd = socket(AF_LOCAL, SOCK_STREAM, 0);
     if(fd >= 0) {
-	if(connect(fd, (struct sockaddr *) &sa, sizeof sa)) {
-	    if((fail_count % 30) == 0)
-		printf(PROGRAM_NAME ": cannot connect socket \"%s\": %s\n",
-		       pathname,
-		       strerror(errno));
-
-	    fail_count++;
-	    close(fd);
-	    return -1;
-	}
-	int flags = fcntl(fd, F_GETFL);
-	fcntl(fd, F_SETFL, flags | O_NONBLOCK);
+        if(connect(fd, (struct sockaddr *) &sa, sizeof sa)) {
+            if((fail_count % 30) == 0) {
+                printf(PROGRAM_NAME ": cannot connect socket \"%s\": %s\n",
+                       pathname,
+                       strerror(errno));
+	    }
+            fail_count++;
+            close(fd);
+            return -1;
+        }
+        int flags = fcntl(fd, F_GETFL);
+        fcntl(fd, F_SETFL, flags | O_NONBLOCK);
     }
     return fd;
 }
@@ -75,7 +75,7 @@ int main(int argc, char * argv[]) {
     int tee_bytes = 0;
 
     if(argc != 3) {
-	error(1, 0, "usage: " PROGRAM_NAME " /path/to/socket cookie-text");
+        error(1, 0, "usage: " PROGRAM_NAME " /path/to/socket cookie-text");
     }
     char * socket_pathname = argv[1];
     char * cookie = argv[2];
@@ -83,8 +83,8 @@ int main(int argc, char * argv[]) {
     char * stop_cookie = malloc(strlen(cookie) + 7);
 
     if(strlen(socket_pathname) > 108) {
-	error(1, 0, "socket pathname \"%s\" is too long, max 108 bytes",
-	      socket_pathname);
+        error(1, 0, "socket pathname \"%s\" is too long, max 108 bytes",
+              socket_pathname);
     };
 
     strcpy(start_cookie, cookie); strcat(start_cookie, " START\n");
@@ -97,46 +97,46 @@ int main(int argc, char * argv[]) {
 
     int quitting = 0;
     while(! quitting) {
-	int nfds = poll(fds, 3, 2000);
-	if(nfds > 0) {
-	    if((fds[0].revents & (POLLIN|POLLHUP)) &&
-	       (out_bytes == 0) &&
-	       (tee_bytes == 0)) {
-		out_bytes = read(fds[0].fd, buf, 8192);
-		if(out_bytes == 0) {
-		    quitting = 1;
-		    buf = PROGRAM_NAME " detected eof of file on stdin, exiting\n";
-		    out_bytes = strlen(buf);
-		};
-		if(is_connected()) tee_bytes = out_bytes;
-	    };
+        int nfds = poll(fds, 3, 2000);
+        if(nfds > 0) {
+            if((fds[0].revents & (POLLIN|POLLHUP)) &&
+               (out_bytes == 0) &&
+               (tee_bytes == 0)) {
+                out_bytes = read(fds[0].fd, buf, 8192);
+                if(out_bytes == 0) {
+                    quitting = 1;
+                    buf = PROGRAM_NAME " detected eof of file on stdin, exiting\n";
+                    out_bytes = strlen(buf);
+                };
+                if(is_connected()) tee_bytes = out_bytes;
+            };
 
-	    if(out_bytes) {
-		out_bytes -= write(fds[1].fd, buf, out_bytes);
-	    };
-	    if(fds[1].revents & (POLLERR|POLLHUP)) {
-		exit(1); // can't even log an error if the logging stream fails
-	    };
-	    if(is_connected()) {
-		if(tee_bytes) {
-		    tee_bytes -= write(fds[2].fd, buf, tee_bytes);
-		};
-		if(fds[2].revents & (POLLERR|POLLHUP)) {
-		    close(fds[2].fd);
-		    fds[2].fd = -1;
-		    (void) write(1, stop_cookie, strlen(stop_cookie));
-		};
-	    };
-	} else {
-	    if(! is_connected()) {
-		fds[2].fd = open_shipper_socket(argv[1]);
-		if(is_connected()) {
-		    /* write cookie to stdout so that the backfill
-		     * process knows we are now logging realtime
-		     */
-		    write(fds[1].fd, start_cookie, strlen(start_cookie));
-		}
-	    }
-	};
+            if(out_bytes) {
+                out_bytes -= write(fds[1].fd, buf, out_bytes);
+            };
+            if(fds[1].revents & (POLLERR|POLLHUP)) {
+                exit(1); // can't even log an error if the logging stream fails
+            };
+            if(is_connected()) {
+                if(tee_bytes) {
+                    tee_bytes -= write(fds[2].fd, buf, tee_bytes);
+                };
+                if(fds[2].revents & (POLLERR|POLLHUP)) {
+                    close(fds[2].fd);
+                    fds[2].fd = -1;
+                    (void) write(1, stop_cookie, strlen(stop_cookie));
+                };
+            };
+        } else {
+            if(! is_connected()) {
+                fds[2].fd = open_shipper_socket(argv[1]);
+                if(is_connected()) {
+                    /* write cookie to stdout so that the backfill
+                     * process knows we are now logging realtime
+                     */
+                    write(fds[1].fd, start_cookie, strlen(start_cookie));
+                }
+            }
+        };
     };
 }