er, don\'t know?

provide stdout to ppp callback scripts
pppd runs them with 0,1,2 => /dev/null but we actually quite like seeing errors in the logs
2024-12-03 19:30:52 +00:00 · 2024-10-17 21:37:08 +01:00 · 2024-10-17 21:35:33 +01:00 · 2024-10-17 21:05:16 +01:00 · 2024-10-16 22:59:01 +01:00 · 2024-10-16 22:56:05 +01:00
176 changed files with 6054 additions and 897 deletions
@@ -103,3 +103,33 @@ a bit more useful :-)
      };
    })
  ];
+
+2024-07-16
+
+* structured parameters are available for the pppoe service
+
+* The "wan" configuration in modules/profiles/gateway.nix has changed:
+instead of passing options that are used to create a pppoe interface,
+callers should create a (pppoe or other) interface and pass that as
+the value of profile.gateway.wan. For the pppoe case this is now only
+very slightly more verbose, and it allows using the gateway profile
+with other kinds of upstream.
+
+2024-8-16
+
+As part of implementing log shipping, the default directory for system
+logs has beenchanged from /run/uncaught-logs to /run/log
+
+2024-10-09
+
+liminix-rebuild is being deprecated. From hereon in, the preferred way
+to do an incremental update on an installed device with a writable
+filesystem is to build the systemConfiguration output
+
+    nix-build  -I liminix-config=hosts/myhost.nix --argstr deviceName turris-omnia -A outputs.systemConfiguration 
+
+and then run the generated `install.sh` script
+
+    result/install.sh root@192.168.8.1
+
+
@@ -89,9 +89,25 @@ in {
      };
    };

+    services.nginx = {
+      enable = true;
+      user = "liminix";
+      virtualHosts.${config.networking.hostName} = {
+        root = "/home/liminix";
+        default = true;
+      };
+    };
+    systemd.services.nginx.serviceConfig.ProtectHome = "read-only";
+
    systemd.services.sshd.wantedBy = pkgs.lib.mkForce [ "multi-user.target" ];

    virtualisation = {
+      forwardPorts = [ {
+        from = "host";
+        host.port = 7654;
+#        guest.address = "10.0.2.15";
+        guest.port =7654;
+      } ];
      qemu = {
        networkingOptions = [ ];
        options =
@@ -114,6 +130,12 @@ in {
        };
      };
    };
+
+    services.tang = {
+      enable = true;
+      ipAddressAllow = [ "10.0.0.0/24" "0.0.0.0/0" ];
+    };
+
    environment.systemPackages =
      let wireshark-nogui = pkgs.wireshark.override { withQt = false ; };
          in with pkgs; [
@@ -124,6 +146,7 @@ in {
            iptables
            usbutils
            busybox
+            clevis
          ];
    security.sudo.wheelNeedsPassword = false;
    networking = {
@@ -1,11 +1,6 @@
-{
-  nixpkgs,
-  unstable,
-  liminix,
-  ...
-}:
 let
-  pkgs = (import nixpkgs { });
+  pkgs = import <nixpkgs> { };
+  liminix = <liminix>;
  borderVmConf = ./bordervm.conf-example.nix;
  inherit (pkgs.lib.attrsets) genAttrs;
  devices = [
@@ -21,7 +16,7 @@ let
  vanilla = ./vanilla-configuration.nix;
  for-device = name:
    (import liminix {
-      inherit nixpkgs borderVmConf;
+      inherit borderVmConf;
      device = import (liminix + "/devices/${name}");
      liminix-config = vanilla;
    }).outputs.default;
@@ -32,7 +27,7 @@ let
    // {
      buildEnv =
        (import liminix {
-          inherit nixpkgs borderVmConf;
+          inherit borderVmConf;
          device = import (liminix + "/devices/qemu");
          liminix-config = vanilla;
        }).buildEnv;
@@ -40,7 +35,7 @@ let
        let
          json =
            (import liminix {
-              inherit nixpkgs borderVmConf;
+              inherit borderVmConf;
              device = import (liminix + "/devices/qemu");
              liminix-config =
                { ... }:
@@ -59,26 +54,26 @@ let
          ];
          src = ./.;
          buildPhase = ''
-            cat ${json} | fennel --correlate doc/parse-options.fnl > doc/modules-generated.rst
-            cat ${json} | fennel --correlate doc/parse-options-outputs.fnl     > doc/outputs-generated.rst
+            cat ${json} | fennel --correlate doc/parse-options.fnl > doc/modules-generated.inc.rst
+            cat ${json} | fennel --correlate doc/parse-options-outputs.fnl     > doc/outputs-generated.inc.rst
            cp ${(import ./doc/hardware.nix)} doc/hardware.rst
            make -C doc html
          '';
          installPhase = ''
            mkdir -p $out/nix-support $out/share/doc/
            cd doc
-            cp *-generated.rst  $out
+            cp *-generated.inc.rst hardware.rst $out
            ln -s ${json} $out/options.json
            cp -a _build/html $out/share/doc/liminix
            echo "file source-dist \"$out/share/doc/liminix\"" \
                > $out/nix-support/hydra-build-products
          '';
      };
-      with-unstable = (import liminix {
-        nixpkgs = unstable;
-        inherit borderVmConf;
-        device = import (liminix + "/devices/qemu");
-        liminix-config = vanilla;
-      }).outputs.default;
    };
-in jobs
+in jobs //
+   {
+     all = pkgs.mkShell {
+       name = "all tests";
+       contents = pkgs.lib.collect pkgs.lib.isDerivation jobs;
+     };
+   }
@@ -2,14 +2,13 @@
  deviceName ? null,
  device ? (import ./devices/${deviceName}),
  liminix-config ? <liminix-config>,
-  nixpkgs ? <nixpkgs>,
  borderVmConf ? ./bordervm.conf.nix,
  imageType ? "primary",
 }:

 let
  overlay = import ./overlay.nix;
-  pkgs = import nixpkgs (
+  pkgs = import <nixpkgs> (
    device.system
    // {
      overlays = [ overlay ];
@@ -34,6 +33,7 @@ let
      ./modules/busybox.nix
      ./modules/hostname.nix
      ./modules/kernel
+      ./modules/klogd.nix
      device.module
      liminix-config
      ./modules/s6
@@ -114,23 +114,38 @@
          eraseBlockSize = 65536;
        };
        rootDevice = "/dev/mtdblock5";
-        dts = {
-          src = "${openwrt.src}/target/linux/ath79/dts/qca9531_glinet_gl-ar750.dts";
-          includes =  [
-            "${openwrt.src}/target/linux/ath79/dts"
-          ];
-        };
+        dts =
+          let
+            upstream = {
+              src = "${openwrt.src}/target/linux/ath79/dts/qca9531_glinet_gl-ar750.dts";
+              includes =  [
+                "${openwrt.src}/target/linux/ath79/dts"
+              ];
+            };
+            cppDtSearchFlags = builtins.concatStringsSep " " (map (f: "-I${f}") upstream.includes);
+            dtcSearchFlags = builtins.concatStringsSep " " (map (f: "-i${f}") upstream.includes);
+
+            patched = pkgs.runCommand "patch-dts" {
+              nativeBuildInputs = with pkgs.pkgsBuildBuild; [pkgs.stdenv.cc dtc ];
+            }
+              ''
+                echo patching dts
+                ${pkgs.stdenv.cc.targetPrefix}cpp -nostdinc -x assembler-with-cpp ${cppDtSearchFlags} -undef -D__DTS__  -o dts.tmp ${upstream.src}
+                dtc -I dts ${dtcSearchFlags} -o $out -O dts dts.tmp
+              '';
+          in upstream // { src = patched; };
+

        networkInterfaces =
          let inherit (config.system.service.network) link;
          in {
            lan = link.build {
              ifname = "lan";
-              devpath = "/devices/platform/ahb/19000000.eth";
+              devpath = "/devices/platform/ahb/1a000000.eth";
            };
            wan = link.build {
              ifname = "wan";
-              devpath = "/devices/platform/ahb/1a000000.eth";
+              devpath = "/devices/platform/ahb/19000000.eth";
            };
            wlan = link.build {
              ifname = "wlan0";
@@ -13,7 +13,7 @@
    GL.iNet GL-MT300N-v2
    ********************

-    The GL-MT300N-v2 "Mango" is is very similar to the :ref:`MT300A <GL.iNet GL-MT300A>, but is
+    The GL-MT300N-v2 "Mango" is is very similar to the :ref:`gl-mt300a`, but is
    based on the MT7628 chipset instead of MT7620.  It's also marginally cheaper
    and comes in a yellow case not a blue one.  Be sure your device is
    v2 not v1, which is a different animal and has only half as much RAM.
@@ -19,7 +19,7 @@
    ARM targets differ from MIPS in that the kernel format expected
    by QEMU is an "Image" (raw binary file) rather than an ELF
    file, but this is taken care of by :command:`run.sh`. Check the
-    documentation for the :ref:`QEMU` (MIPS) target for more information.
+    documentation for the :ref:`qemu` target for more information.

  '';

@@ -4,143 +4,134 @@ System Administration
 Services on a running system
 ****************************

-* add an s6-rc cheatsheet here
+Liminix services are built on s6-rc, which is itself layered on s6.
+Services are defined at build time in your configuration (see
+:ref:`configuration-services` for information) and can't be added
+to/changed at runtime, but to monitor
+events or diagnose problems you may need to inspect them on the
+running system. Here are some of the most commonly used s6,-rc
+commands:

+.. list-table:: Service management quick reference
+   :widths: 55 45
+   :header-rows: 1
+   
+   * - What
+     - How
+   * - List all running services
+     - ``s6-rc -a list``
+   * - List all services that are **not** running
+     - ``s6-rc -da list``
+   * - List services that ``wombat`` depends on
+     - ``s6-rc-db dependencies wombat``
+   * - ... transitively
+     - ``s6-rc-db all-dependencies wombat``
+   * - List services that depend on service ``wombat``
+     - ``s6-rc-db -d dependencies wombat``
+   * - ... transitively
+     - ``s6-rc-db -d all-dependencies wombat``
+   * - Stop service ``wombat`` and everything depending on it
+     - ``s6-rc -d change wombat``
+   * - Start service ``wombat`` (but not any services depending on it)
+     - ``s6-rc -u change wombat``
+   * - Start service ``wombat`` and all* services depending on it
+     - ``s6-rc-up-tree wombat``

-Flashing and updating
-*********************
+:command:`s6-rc-up-tree` brings up a service and all services that
+depend on it, except for any services that depend on a "controlled"
+service that is not currently running. Controlled services are not
+started at boot time but in response to external events (e.g. plugging
+in a particular piece of hardware) so you probably don't want to be
+starting them by hand if the conditions aren't there.

+A service may be **up** or **down** (there are no intermediate states
+like "started" or "stopping" or "dying" or "cogitating"). Some (but
+not all) services have "readiness" notifications: the dependents of a
+service with a readiness notification won't be started until the
+service signals (by writing to a nominated file descriptor) that it's
+prepared to start work. Most services defined by Liminix also have a
+``timeout-up`` parameter, which means that if a service has readiness
+notifications and doesn't become ready in the allotted time (defaults
+20 seconds) it will be terminated and its state set to **down**.

+If the process providing a service dies, it will be restarted
+automatically. Liminix does not automatically set it to **down**.

-Flashing from Liminix
-=====================
+(If the process providing a service dies without ever notifying
+readiness, Liminix will restart it as many times as it has to until the
+timeout period elapses, and then stop it and mark it down.)

-The flash procedure from an existing Liminix-system has two steps.
-First we reboot the device (using "kexec") into an "ephemeral"
-RAM-based version of the new configuration, then when we're happy it
-works we can flash the image - and if it doesn't work we can reboot
-the device again and it will boot from the old image.
+Controlled services
+===================

+**Controlled** services are those which are started/stopped on demand
+by a **controller** (another service) instead of being started at boot
+time.  For example:

-Building the RAM-based image
----------------------------
+* ``svc.uevent-rule.build`` creates a controlled service which is
+  active when a particular hardware device (identified by uevent/sysfs
+  directory) is present.

-To create the ephemeral image, build ``outputs.kexecboot`` instead of
-``outputs.default``.  This generates a directory containing the root
-filesystem image and kernel, along with an executable called `kexec`
-and a `boot.sh` script that runs it with appropriate arguments.
+* ``svc.round-robin.build`` creates a service controller that
+  invokes two or more services in turn, running the next one when the
+  process providing the previous one exits. We use this for failover
+  from one network connection to a backup connection, for example.

-For example
+* ``svc.health-check.build`` creates a service controller that
+  runs a controlled service and periodically tests whether it is
+  healthy by running an external health check command or script. If the
+  check command repeatedly fails, the controlled service is
+  restarted.
+
+  The Configuration section of the manual describes controlled
+  services in more detail. Some operational considerations
+
+* ``round-robin`` detects a service status by looking at its
+  :file:`outputs` directory, so it won't work unless the service
+  creates some outputs. This is considered a bug and will be
+  fixed in a future release
+
+* ``health-check`` works for longruns but not for oneshots, as it
+  internally relies on ``s6-svc`` to restart the process
+
+Logs
+====
+
+Logs for all services are collated into :file:`/run/log/current`.
+The log file is rotated when it reaches a threshold size, into another
+file in the same directory whose name contains a TAI64 timestamp.
+
+Each log line is prefixed with a TAI64 timestamp and the name of the
+service, if it is a longrun. If it is a oneshot, a timestamp and the
+name of some other service. To convert the timestamp into a
+human-readable format, use :command:`s6-tai64nlocal`.

 .. code-block:: console

-   nix-build -I liminix-config=./examples/arhcive.nix \
-     --arg device "import ./devices/gl-ar750"
-     -A outputs.kexecboot && \
-     (tar chf - result | ssh root@the-device tar -C /run -xvf -)
-
-and then login to the device and run
-
-.. code-block:: console
-
-   cd /run/result
-   sh ./boot.sh .
+  # ls -l /run/log/
+  -rw-r--r--    1         0 lock
+  -rw-r--r--    1         0 state
+  -rwxr--r--    1     98059 @4000000000025cb629c311ac.s
+  -rwxr--r--    1     98061 @40000000000260f7309c7fb4.s
+  -rwxr--r--    1     98041 @40000000000265233a6cc0b6.s
+  -rwxr--r--    1     98019 @400000000002695d10c06929.s
+  -rwxr--r--    1     98064 @4000000000026d84189559e0.s
+  -rwxr--r--    1     98055 @40000000000271ce1e031d91.s
+  -rwxr--r--    1     98054 @400000000002760229733626.s
+  -rwxr--r--    1     98104 @4000000000027a2e3b6f4e12.s
+  -rwxr--r--    1     98023 @4000000000027e6f0ed24a6c.s
+  -rw-r--r--    1     42374 current
+  
+  # tail -2 /run/log/current
+  @40000000000284f130747343 wan.link.pppoe Connect: ppp0 <--> /dev/pts/0
+  @40000000000284f230acc669 wan.link.pppoe sent [LCP ConfReq id=0x1 <asyncmap 0x0> <magic 0x667a9594> <pcomp> <accomp>]
+  # tail -2 /run/log/current  | s6-tai64nlocal
+  1970-01-02 21:51:45.828598156 wan.link.pppoe sent [LCP ConfReq id=0x1 <asyncmap 0x0> <magic 0x667a9594> <pcomp> <accom
+  p>]
+  1970-01-02 21:51:48.832588765 wan.link.pppoe sent [LCP ConfReq id=0x1 <asyncmap 0x0> <magic 0x667a9594> <pcomp> <accom
+  p>]


-This will load the new kernel and map the root filesystem into a RAM
-disk, then start executing the new kernel. *This is effectively a
-reboot - be sure to close all open files and finish anything else
-you were doing first.*
-
-If the new system crashes or is rebooted, then the device will revert
-to the old configuration it finds in flash.
-
-
-Building the second (permanent) image
-------------------------------------
-
-While running in the kexecboot system, you can build the permanent
-image and copy it to the device with :command:`ssh`
-
-.. code-block:: console
-
-   build-machine$ nix-build -I liminix-config=./examples/arhcive.nix \
-     --arg device "import ./devices/gl-ar750"
-     -A outputs.default && \
-     (tar chf - result | ssh root@the-device tar -C /run -xvf -)
-
-   build-machine$ tar chf - result/firmware.bin | \
-    ssh root@the-device tar -C /run -xvf -
-
-Next you need to connect to the device and locate the "firmware"
-partition, which you can do with a combination of :command:`dmesg`
-output and the contents of :file:`/proc/mtd`
-
-.. code-block:: console
-
-   <5>[    0.469841] Creating 4 MTD partitions on "spi0.0":
-   <5>[    0.474837] 0x000000000000-0x000000040000 : "u-boot"
-   <5>[    0.480796] 0x000000040000-0x000000050000 : "u-boot-env"
-   <5>[    0.487056] 0x000000050000-0x000000060000 : "art"
-   <5>[    0.492753] 0x000000060000-0x000001000000 : "firmware"
-
-   # cat /proc/mtd
-   dev:    size   erasesize  name
-   mtd0: 00040000 00001000 "u-boot"
-   mtd1: 00010000 00001000 "u-boot-env"
-   mtd2: 00010000 00001000 "art"
-   mtd3: 00fa0000 00001000 "firmware"
-   mtd4: 002a0000 00001000 "kernel"
-   mtd5: 00d00000 00001000 "rootfs"
-
-Now run (in this example)
-
-.. code-block:: console
-
-   flashcp -v firmware.bin /dev/mtd3
-
-
-"I know my new image is good, can I skip the intermediate step?"
----------------------------------------------------------------
-
-In addition to giving you a chance to see if the new image works, this
-two-step process ensures that you're not copying the new image over
-the top of the active root filesystem. Sometimes it works, but you
-will at least need physical access to the device to power-cycle it
-because it will be effectively frozen afterwards.
-
-
-Flashing from the boot monitor
-==============================
-
-If you are prepared to open the device and have a TTL serial adaptor
-of some kind to connect it to, you can probably use U-Boot and a TFTP
-server to download and flash the image.  This is quite
-hardware-specific, and sometimes involves soldering: please refer
-to :ref:`serial`.
-
-
-Flashing from OpenWrt
-=====================
-
-.. CAUTION:: Untested! A previous version of these instructions
-	     (without the -e flag) led to bricking the device
-	     when flashing a jffs2 image. If you are reading
-	     this message, nobody has yet reported on whether the
-	     new instructions are any better.
-
-If your device is running OpenWrt then it probably has the
-:command:`mtd` command installed. After transferring the image onto the
-device using e.g. :command:`ssh`,  you can run it as follows:
-
-.. code-block:: console
-
-   mtd -e -r write /tmp/firmware.bin firmware
-
-The options to this command are for "erase before writing" and "reboot
-after writing".
-
-For more information, please see the `OpenWrt manual <https://openwrt.org/docs/guide-user/installation/sysupgrade.cli>`_ which may also contain (hardware-dependent) instructions on how to flash an image using the vendor firmware - perhaps even from a web interface.

 Updating an installed system (JFFS2)
 ************************************
@@ -165,20 +156,37 @@ Note that this only copies the package to the device: it doesn't update
 any profile to add it to ``$PATH``


+.. _rebuilding the system:
+
 Rebuilding the system
 =====================

-:command:`liminix-rebuild` is the Liminix analogue of :command:`nixos-rebuild`, although its operation is a bit different because it expects to run on a build machine and then copy to the host device. Run it with the same ``liminix-config`` and ``device`` parameters as you would run :command:`nix-build`, and it will build any new/changed packages and then copy them to the device using SSH. For example:
+Liminix has a mechanism for in-place updates of a running system which
+is analogous to :command:`nixos-rebuild`, but its operation is a
+bit different because it expects to run on a build machine and then
+copy to the host device. To use this, build the `outputs.systemConfiguration`
+target and then run the :command:`result/install.sh` script it generates.

 .. code-block:: console

-     liminix-rebuild root@the-device  -I liminix-config=./examples/rotuer.nix --arg device "import ./devices/gl-ar750"
+    nix-build -I liminix-config=./my-configuration.nix \
+       --arg device "import ./devices/mydevice" \
+       -A outputs.systemConfiguration
+    ./result/install.sh root@the-device 

-This will
+The install script uses min-copy-closure to copy new or changed
+packages to the device, then (perhaps) reboots it. The reboot
+behaviour can be affected by flags:
+
+* `--no-reboot` will cause it not to reboot at all, if you would
+  rather do that yourself. Note that none of the newly-installed or
+  updated services will be running until you do.
+
+* `--fast` causes it tn not do a full reboot, but instead to restart
+  only the services that have been changed. This will restart all of
+  the services that have updated store paths (and anything that
+  depends on them), but will not affect services that haven't changed.

-* build anything that needs building
-* copy new or changed packages to the device
-* reboot the device

 It doesn't delete old packages automatically: to do that run
 :command:`min-collect-garbage`, which will delete any packages not in
@@ -196,3 +204,116 @@ Caveats
  nixpkgs).

 * it cannot upgrade the kernel, only userland
+
+.. _levitate:  
+
+Reinstalling on a running system
+********************************
+
+Liminix is initially installed from a monolithic
+:file:`firmware.bin` - and unless you're running a writable
+filesystem, the only way to update it is to build and install a whole
+new :file:`firmware.bin`.  However, you probably would prefer not to
+have to remove it from its installation site, unplug it from the
+network and stick serial cables in it all over again.
+
+It is not (generally) safe to install a new firmware onto the flash
+partitions that the active system is running on. To address this we
+have :command:`levitate`, which a way for a running Liminix system to
+"soft restart" into a ramdisk running only a limited set of services,
+so that the main partitions can then be safely flashed.
+
+
+
+Configuration
+=============
+
+Levitate *needs to be configured when you create the initial system*
+to specify which services/packages/etc to run in maintenance
+mode. Most likely you want to configure a network interface and an ssh
+for example so that you can login to reflash it.
+
+.. code-block:: nix
+
+  defaultProfile.packages = with pkgs; [
+    ...
+    (levitate.override {
+      config  = {
+        services = {
+          inherit (config.services) dhcpc sshd watchdog;
+        };
+        defaultProfile.packages = [ mtdutils ];
+        users.root = config.users.root;
+      };
+    })
+  ];
+  		
+
+
+Use
+===
+
+Connect (with ssh, probably) to the running Liminix system that you
+wish to upgrade.
+
+.. code-block:: console
+
+  bash$ ssh root@the-device
+  
+Run :command:`levitate`. This takes a little while (perhaps a few
+tens of seconds) to execute, and copies all config required for
+maintenance mode to :file:`/run/maintenance`.
+  
+.. code-block:: console
+  
+  # levitate 
+  
+Reboot into maintenance mode. You will be logged out
+  
+.. code-block:: console
+
+  # reboot
+  
+Connect to the device again - note that the ssh host key will have changed.
+  
+.. code-block:: console
+
+  # ssh -o UserKnownHostsFile=/dev/null root@the-device
+  
+Check we're in maintenance mode
+  
+.. code-block:: console
+
+  # cat /etc/banner 
+  
+  LADIES AND GENTLEMEN WE ARE FLOATING IN SPACE
+  
+  Most services are disabled. The system is operating
+  with a ram-based root filesystem, making it safe to
+  overwrite the flash devices in order to perform
+  upgrades and maintenance.
+  
+  Don't forget to reboot when you have finished.
+  
+Perform the upgrade, using flashcp. This is an example,
+your device will differ
+   
+.. code-block:: console
+
+  # cat /proc/mtd 
+  dev:    size   erasesize  name
+  mtd0: 00030000 00010000 "u-boot"
+  mtd1: 00010000 00010000 "u-boot-env"
+  mtd2: 00010000 00010000 "factory"
+  mtd3: 00f80000 00010000 "firmware"
+  mtd4: 00220000 00010000 "kernel"
+  mtd5: 00d60000 00010000 "rootfs"
+  mtd6: 00010000 00010000 "art"
+  # flashcp -v firmware.bin mtd:firmware
+
+All done
+  
+.. code-block:: console
+
+  # reboot
+
@@ -7,19 +7,19 @@
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information

 project = 'Liminix'
-copyright = '2023, Daniel Barlow'
+copyright = '2023-2024 Daniel Barlow'
 author = 'Daniel Barlow'

 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

 extensions = [
-    'sphinx.ext.autosectionlabel'
+#    'sphinx.ext.autosectionlabel'
 ]
 autosectionlabel_prefix_document = True

 templates_path = ['_templates']
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ['*.inc.rst', '_build', 'Thumbs.db', '.DS_Store']



@@ -1,27 +1,8 @@
+.. _configuration:
+
 Configuration
 #############

-Liminix uses the Nix language to provide congruent configuration
-management.  This means that to change anything about the way in
-which a Liminix system works, you make that change in
-your :file:`configuration.nix` (or one of the other files it references),
-and rerun :command:`nix-build` or :command:`liminix-rebuild` to action
-the change. It is not possible (at least, without shenanigans) to make
-changes by logging into the device and running imperative commands
-whose effects may later be overridden: :file:`configuration.nix`
-always describes the entire system and can be used to recreate that
-system at any time.  You can usefully keep it under version control.
-
-If you are familiar with NixOS, you will notice some similarities
-between NixOS and Liminix configuration, and also some
-differences. Sometimes the differences are due to the
-resource-constrained devices we deploy onto, sometimes due to
-differences in the uses these devices are put to.
-
-
-Configuration taxonomy
-**********************
-
 There are many things you can specify in a configuration, but these
 are the ones you most commonly need to change:

@@ -86,14 +67,166 @@ domains, or you want to run two SSH daemons on different ports.
 	 don't use the NixOS modules themselves, because the
 	 underlying system is not similar enough for them to work.

+.. _configuration-services:

 Services
 ********

-We use the `s6-rc service manager <https://www.skarnet.org/software/s6-rc/overview.html>`_  to start/stop/restart services and handle
-service dependencies. Any attribute in `config.services` will become
-part of the default set of services that s6-rc will try to bring up on
-boot.
+In Liminix a service is any kind of long-running task or process on
+the system, that is managed (started, stopped, and monitored) by a
+service supervisor.  A typical SOHO router might have services to
+
+* answer DHCP and DNS requests from the LAN
+* provide a wireless access point
+* connect using PPPoE or L2TP to an upstream network
+* start/stop the firewall
+* enable/disable IP packet forwarding
+* mount filesystems
+
+(Some of these might not be considered services using other
+definitions of the term: for example, this L2TP process would be a
+"client" in the client/server classification; and enabling packet
+forwarding doesn't require any long-lived process - just a setting to
+be toggled.  However, there is value in being able to use the same
+abstractions for all the things to manage them and specify their
+dependency relationships - so in Liminix "everything is a service")
+
+The service supervision system enables service health monitoring,
+restart of unhealthy services, and failover to "backup" services when
+a primary service fails or its dependencies are unavailable. The
+intention is that you have a framework in which you can specify policy
+requirements like "ethernet wan dhcp-client should be restarted if it
+crashes, but if it can't start because the hardware link is down, then
+4G ppp service should be started instead".
+
+Any attribute in `config.services` will become part of the default set
+of services that s6-rc will try to bring up.  Services are usually
+started at boot time, but **controlled services** are those that are
+required only in particular contexts.  For example, a service to mount
+a USB backup drive should run only when the drive is attached to the
+system. Liminix currently implements three kinds of controlled service:
+
+* "uevent-rule" service controllers use sysfs/uevent to identify when
+  particular hardware devices are present, and start/stop a controlled
+  service appropriately.
+
+* the "round-robin" service controller is used for service failover:
+  it allows you to specify a list of services and runs each of them
+  in turn until it exits, then runs the next.
+
+* the "health-check" service wraps another service, and runs a "health
+  check" command at regular intervals. When the health check fails,
+  indicating that the wrapped service is not working, it is terminated
+  and allowed to restart.
+
+Runtime secrets (external vault)
+================================
+
+Secrets (such as wifi passphrases, PPP username/password, SSH keys,
+etc) that you provide as literal values in :file:`configuration.nix`
+are processed into into config files and scripts at build time, and
+eventually end up in various files in the (world-readable)
+:file:`/nix/store` before being baked into a flashable image. To
+change a secret - whether due to a compromise, or just as part of to a
+routine key rotation - you need to rebuild the configuration and
+potentially reflash the affected devices.
+
+To avoid this, you may instead use a "secrets service", which is a
+mechanism for your device to fetch secrets from a source external to
+the Nix store, and create at runtime the configuration files and
+scripts that start the services which require them.
+
+Not every possible parameter to every possible service is configurable
+using a secrets service. Parameters which can be configured this way
+are those with the type ``liminix.lib.types.replacable``. At the time
+this document was written, these include:
+
+* ppp (pppoe and l2tp): ``username``, ``password``
+* ssh: ``authorizedKeys``
+* hostapd: all parameters (most likely to be useful for ``wpa_passphrase``)
+
+To use a runtime secret for any of these parameters:
+
+* create a secrets service to specify the source of truth for secrets
+* use the :code:`outputRef` function in the service parameter to specify the secrets service and path
+
+For example, given you had an HTTPS server hosting a JSON file with the structure
+
+.. code-block:: json
+
+    "ssh": {
+      "authorizedKeys": {
+	"root": [ "ssh-rsa ....",  "ssh-rsa ....", ... ]
+	"guest": [ "ssh-rsa ....",  "ssh-rsa ....", ... ]
+      }
+    }
+ 
+you could use a :file:`configuration.nix` fragment something like this
+to make those keys visible to ssh:
+  
+.. code-block:: nix  
+
+    services.secrets = svc.secrets.outboard.build {
+      name = "secret-service";
+      url = "http://10.0.0.1/secrets.json";
+      username = "secrets";
+      password = "liminix";
+      interval = 30; # minutes
+      dependencies = [ config.services.lan ];
+    };
+    services.sshd = svc.ssh.build {
+      authorizedKeys = outputRef config.services.secrets "ssh/authorizedKeys";
+    };
+
+
+    
+There are presently two implementations of a secrets service:
+
+Outboard secrets (HTTPS)
+------------------------
+
+This service expects a URL to a JSON file containing all the secrets.
+
+You may specify a username and password along with the URL, which are
+used if the file is password-protected (HTTP Basic
+authentication). Note that this is not a protection against a
+malicious local user: the username and password are normal build-time
+parameters so will be readable in the Nix store.  This is a mitigation
+against the URL being accidentally discovered due to e.g. a log file
+or error message on the server leaking.
+
+
+Tang secrets (encrypted local file)
+-----------------------------------
+
+Aternatively, secrets may be stored locally on the device, in a file
+that has been encrypted using `Tang <https://github.com/latchset/tang>`_.
+
+    Tang is a server for binding data to network presence.
+
+    This sounds fancy, but the concept is simple. You have some data, but you only want it to be available when the system containing the data is on a certain, usually secure, network. 
+
+
+.. code-block:: nix
+
+    services.secrets = svc.secrets.tang.build {
+      name = "secret-service";
+      path = "/run/mnt/usbstick/secrets.json.jwe";
+      interval = 30; # minutes
+      dependencies = [ config.services.mount-usbstick ];
+    };
+
+The encryption uses the
+same scheme/algorithm as `Clevis <https://github.com/latchset/clevis>`_ : you may use the `Clevis instructions <https://github.com/latchset/clevis?tab=readme-ov-file#pin-tang>`_ to
+encrypt the file on another host and then copy it to your Liminix
+device, or you can use :command:`tangc encrypt` to encrypt directly on
+the device.  (That latter approach may pose a chicken/egg problem if
+the device needs secrets to boot up and run the services you are
+relying on in order to login).
+ 
+
+Writing services
+================

 For the most part, for common use cases, hopefully the services you
 need will be defined by modules and you will only have to pass the
@@ -141,11 +274,101 @@ Services may have dependencies: as you see above in the ``cowsayd``
 example, it depends on some service called ``config.services.lan``,
 meaning that it won't be started until that other service is up.

-..
-	TODO: explain service outputs
+Service outputs
+===============
+
+Outputs are a mechanism by which a service can provide data which may
+be required by other services. For example:
+
+* the DHCP client service can expect to receive nameserver address
+  information as one of the fields in the response from the DHCP
+  server: we provide that as an output which a dependent service for a
+  stub name resolver can use to configure its upstream servers.
+
+* a service that creates a new network interface (e.g. ppp) will
+  provide the name of the interface (:code:`ppp0`, or :code:`ppp1` or
+  :code:`ppp7`) as an output so that a dependent service can reference
+  it to set up a route, or to configure firewall rules.
+
+A service :code:`myservice` should write its outputs as files in
+:file:`/run/services/outputs/myservice`: you can look around this
+directory on a running Liminix system to see how it's used currently.
+Usually we use the :code:`in_outputs` shell function in the
+:command:`up` or :command:`run` attributes of the service:
+
+.. code-block:: shell
+		
+    (in_outputs ${name}
+     for i in lease mask ip router siaddr dns serverid subnet opt53 interface ; do
+       (printenv $i || true) > $i
+     done)
+ 
+The outputs are just files, so technically you can read them using
+anything that can read a file. Liminix has two "preferred"
+mechanisms, though:
+
+One-off lookups
+---------------
+
+In any context that ends up being evaluated by the shell, use 
+:code:`output` to print the value of an output
+
+.. code-block:: nix
+		
+    services.defaultroute4 = svc.network.route.build {
+      via = "$(output ${services.wan} address)";
+      target = "default";
+      dependencies = [ services.wan ];
+    };	  
+
+    
+Continuous updates
+------------------
+
+The downside of using shell functions in downstream service startup
+scripts is that they only run when the service starts up: if a service
+output *changes*, the downstream service would have to be restarted to
+notice the change. Sometimes this is OK but other times the downstream
+has no other need to restart, if it can only get its new data.
+
+For this case, there is the :code:`anoia.svc` Fennel library, which
+allows you to write a simple loop which is iterated over whenever a
+service's outputs change. This code is from
+:file:`modules/dhcp6c/acquire-wan-address.fnl`
+
+.. code-block:: fennel
+
+    (fn update-addresses [wan-device addresses new-addresses exec]
+      ;; run some appropriate "ip address [add|remove]" commands
+      )
+
+    (fn run []
+      (let [[state-directory wan-device] arg
+	    dir (svc.open state-directory)]
+	(accumulate [addresses []
+		     v (dir:events)]
+	  (update-addresses wan-device addresses
+	                    (or (v:output "address") []) system))))		
+
+The :code:`output` method seen here accepts a filename (relative
+to the service's output directory), or a directory name. It 
+returns the first line of that file, or for directories it
+returns a table (Lua's key/value datastructure, similar to
+a hash/dictionary) of the outputs in that directory.
+	  
+    
+Output design considerations
+----------------------------
+
+For preference, outputs should be short and simple, and not require
+downstream services to do complicated parsing in order to use them.
+Shell commands in Liminix are run using the Busybox shell which
+doesn't have the niceties of an advanced shell like Bash let alone
+those of a real programming language.
+
+Note also that the Lua :code:`svc` library only reads the first line
+of each output.

-..
-	TODO: outputs that change, and services that poll other services

 Module implementation
 *********************
@@ -192,7 +415,7 @@ To expose a service template in a module, it needs the following:

 .. code-block:: nix

-    config.system.service.cowsay = liminix.callService ./service.nix {
+    config.system.service.cowsay = config.system.callService ./service.nix {
      address = mkOption {
 	type = types.str;
 	default = "0.0.0.0";
@@ -88,64 +88,6 @@ time with configurations for RP-PPPoE and/or Accel PPP.`
 Hardware devices
 ****************

-.. _serial:
-
-U-Boot and serial shenanigans
-=============================
-
-Every device that we have so far encountered in Liminix uses `U-Boot,
-the "Universal Boot Loader" <https://docs.u-boot.org/en/latest/>`_ so
-it's worth knowing a bit about it. "Universal" is in this context a
-bit of a misnomer, though: encountering *mainline* U-Boot is very rare
-and often you'll find it is a fork from some version last updated
-in 2008. Upgrading U-Boot is more or less complicated depending on the
-device and is outside scope for Liminix.
-
-To speak to U-Boot on your device you'll usually need a serial
-connection to it.  This is device-specific. Usually it involves
-opening the box, locating the serial header pins (TX, RX and GND) and
-connecting a USB TTL converter to them.
-
-The Rolls Royce of USB/UART cables is the `FTDI cable
-<https://cpc.farnell.com/ftdi/ttl-232r-rpi/cable-debug-ttl-232-usb-rpi/dp/SC12825?st=usb%20to%20uart%20cable>`_,
-but there are cheaper alternatives based on the PL2303 and CP2102 chipsets.  Or
-get creative and use the `UART GPIO pins <https://pinout.xyz/>`_ on a Raspberry Pi. Whatever you do, make sure
-that the voltages are compatible: if your device is 3.3V (this is
-typical but not universal), you don't want to be sending it 5v or
-(even worse) 12v.
-
-Run a terminal emulator such as Minicom on the computer at other end
-of the link. 115200 8N1 is the typical speed.
-
-.. NOTE::
-
-   TTL serial connections typically have no form of flow control and
-   so don't always like having massive chunks of text pasted into
-   them - and U-Boot may drop characters while it's busy. So don't
-   necessarily expect to copy-paste large chunks of text into the
-   terminal emulator and have it work just like that.
-
-   If using Minicom, you may find it helps to bring up the "Termimal
-   settings" dialog (C^A T), then configure "Newline tx delay" to
-   some small but non-zero value.
-
-When you turn the router on you should be greeted with some messages
-from U-Boot, followed by the instruction to hit some key to stop
-autoboot. Do this and you will get to the prompt. If you didn't see
-anything, the strong likelihood is that TX and RX are the wrong way
-around. If you see garbage, try a different speed.
-
-Interesting commands to try first in U-Boot are :command:`help` and
-:command:`printenv`.
-
-To do anything useful with U-Boot you will probably need a way to get
-large binary files onto the device, and the usual way to do this is by
-adding a network connection and using TFTP to download them. It's
-quite common that the device's U-Boot doesn't speak DHCP so it will
-need a static LAN address. You might also want to keep it away from
-your "real" LAN: see :ref:`bng` for some potentially useful tooling
-to use it on an isolated network.
-

 TFTP
 ====
@@ -290,18 +232,13 @@ Running tests
 *************

 You can run all of the tests by evaluating :file:`ci.nix`, which is the
-input I use in Hydra. Note that it expects Nixpkgs stable `and` unstable
-as inputs, because it builds the qemu device against both.
+input I use in Hydra. 

 .. code-block:: console

-    nix-build --argstr liminix `pwd`  --arg  nixpkgs "<nixpkgs>" \
-     --argstr unstable `pwd`/../unstable-nixpkgs/ ci.nix
-
-To run a single named test, use the ``-A`` flag. For example, ``-A pppoe``
-
-
-
+    nix-build -I liminix=`pwd`  ci.nix -A pppoe # run one job
+    nix-build -I liminix=`pwd`  ci.nix -A all # run all jobs
+    

 Troubleshooting
 ***************
@@ -7,11 +7,15 @@ let
    n:
    let
      d = import ../devices/${n}/default.nix;
+      tag = ".. _${lib.strings.replaceStrings [" "] ["-"] n}:";
      d' = {
-        description = "${n}\n${substring 0 (stringLength n) "********************************"}\n";
+        description = ''
+          ${n}
+          ${substring 0 (stringLength n) "********************************"}
+        '';
      } // d;
    in
-    d'.description
+      "${tag}\n\n${d'.description}"
  ) devices;
 in
 writeText "hwdoc" ''
@@ -7,6 +7,7 @@ Liminix

   intro
   tutorial
+   installation
   configuration
   admin
   development
@@ -0,0 +1,211 @@
+Installation
+############
+
+Hardware devices vary wildly in their affordances for installing new
+operating systems, so it should be no surprise that the Liminix
+installation procedure is hardware-dependent. This section contains
+generic instructions, but please refer to the documentation for your
+device to find whether and how well they apply.
+
+
+Building a firmware image
+*************************
+
+Liminix uses the Nix language to provide congruent configuration
+management.  This means that to change anything about the way in
+which a Liminix system works, you make that change in
+your :file:`configuration.nix` (or one of the other files it references),
+and rerun :command:`nix-build` to action
+the change. It is not possible (at least, without shenanigans) to make
+changes by logging into the device and running imperative commands
+whose effects may later be overridden: :file:`configuration.nix`
+always describes the entire system and can be used to recreate that
+system at any time.  You can usefully keep it under version control.
+
+If you are familiar with NixOS, you will notice some similarities
+between NixOS and Liminix configuration, and also some
+differences. Sometimes the differences are due to the
+resource-constrained devices we deploy onto, sometimes due to
+differences in the uses these devices are put to.
+
+For a more full description of how to configure Liminix, see
+:ref:`configuration`. Assuming for the moment that you want a typical
+home wireless gateway/router, the best way to get started is to copy
+:file:`examples/rotuer.nix` and edit it for your requirements.
+
+
+.. code-block:: console
+
+    $ cp examples/rotuer.nix configuration.nix
+    $ vi configuration.nix # other editors are available 
+    $ # adjust this next command for your hardware device
+    $ nix-build -I liminix-config=./configuration.nix \
+     --arg device "import ./devices/gl-mt300a" -A outputs.default
+
+Usually  (not always, *please check the documentation for your device*)
+this will leave you with a file :file:`result/firmware.bin`
+which you now need to flash to the device.
+
+
+Flashing from the boot monitor (TFTP install)
+*********************************************
+
+If you are prepared to open the device and have a TTL serial adaptor
+of some kind to connect it to, you can probably use U-Boot and a TFTP
+server to download and flash the image.
+
+This is quite hardware-specific and may even involve soldering - see
+the documention for your device. However, it is in some ways the most
+"reliable" option: if you can see what's happening (or not happening)
+in early boot, the risk of "bricking" is substantially reduced and you
+have options for recovering if you misstep or flash a bad image.
+
+
+.. _serial:
+
+U-Boot and serial shenanigans
+=============================
+
+Every device that we have so far encountered in Liminix uses `U-Boot,
+the "Universal Boot Loader" <https://docs.u-boot.org/en/latest/>`_ so
+it's worth knowing a bit about it. "Universal" is in this context a
+bit of a misnomer, though: encountering *mainline* U-Boot is very rare
+and often you'll find it is a fork from some version last updated
+in 2008. Upgrading U-Boot is more or less complicated depending on the
+device and is outside scope for Liminix.
+
+To speak to U-Boot on your device you'll usually need a serial
+connection to it.  This typically involves opening the box, locating
+the serial header pins (TX, RX and GND) and connecting a USB TTL
+converter to them.
+
+The Rolls Royce of USB/UART cables is the `FTDI cable
+<https://cpc.farnell.com/ftdi/ttl-232r-rpi/cable-debug-ttl-232-usb-rpi/dp/SC12825?st=usb%20to%20uart%20cable>`_,
+but there are cheaper alternatives based on the PL2303 and CP2102 chipsets -   or you could even 
+get creative and use the `UART GPIO pins <https://pinout.xyz/>`_ on a Raspberry Pi. Whatever you do, make sure
+that the voltages are compatible: if your device is 3.3V (this is
+typical but not universal), you don't want to be sending it 5v or
+(even worse) 12v.
+
+Run a terminal emulator such as Minicom on the computer at other end
+of the link. 115200 8N1 is the typical speed.
+
+.. NOTE::
+
+   TTL serial connections often have no flow control and
+   so don't always like having massive chunks of text pasted into
+   them - and U-Boot may drop characters while it's busy. So don't
+   do that.
+
+   If using Minicom, you may find it helps to bring up the "Termimal
+   settings" dialog (C^A T), then configure "Newline tx delay" to
+   some small but non-zero value.
+
+When you turn the router on you should be greeted with some messages
+from U-Boot, followed by the instruction to hit some key to stop
+autoboot. Do this and you will get to the prompt. If you didn't see
+anything, the strong likelihood is that TX and RX are the wrong way
+around. If you see garbage, try a different speed.
+
+Interesting commands to try first in U-Boot are :command:`help` and
+:command:`printenv`.
+
+You will also need to configure a TFTP server on a network that's
+accessible to the device: how you do that will vary according to which
+TFTP server you're using and so is out of scope for this document.
+
+
+
+Buildiing and installing the image
+==================================
+
+Follow the device-specific instructions for "TFTP install": usually,
+the steps are 
+
+* build the `outputs.mtdimage` output
+* copy :file:`result/firmware.bin` to your TFTP server
+* copy/paste the commands in :file:`result/flash.scr` one at a time into the U-Boot command line
+* reset the device
+
+You should now see messages from U-Boot, then from the Linux kernel
+and eventually a shell prompt.
+
+.. NOTE:: Before you reboot, check which networks the device is
+          plugged into, and disconnect as necessary. If you've just
+          installed a DHCP server or anything similar that responds to
+          broadcasts, you may not want it to do that on the network
+          that you temporarily connected it to for installing it.
+
+
+
+Flashing from OpenWrt
+*********************
+
+.. CAUTION:: Untested! A previous version of these instructions
+	     (without the -e flag) led to bricking the device
+	     when flashing a jffs2 image. If you are reading
+	     this message, nobody has yet reported on whether the
+	     new instructions are any better.
+
+If your device is running OpenWrt then it probably has the
+:command:`mtd` command installed. Build the `outputs.mtdimage` output
+(as you would for a TFTP install) and then transfer
+:file:`result/firmware.bin` onto the device using e.g.
+:command:`scp`. Now flash as follows:
+
+.. code-block:: console
+
+   mtd -e -r write /tmp/firmware.bin firmware
+
+The options to this command are for "erase before writing" and "reboot
+after writing".
+
+For more information, please see the `OpenWrt manual <https://openwrt.org/docs/guide-user/installation/sysupgrade.cli>`_ which may also contain (hardware-dependent) instructions on how to flash an image using the vendor firmware - perhaps even from a web interface.
+
+
+Flashing from Liminix
+*********************
+
+If the device is already running Liminix and has been configured with
+:command:`levitate`, you can use that to safely flash your new image.
+Refer to :ref:`levitate` for an explanation.
+
+If the device is running Liminix but doesn't have :command:`levitate`
+your options are more limited. You may attempt to use
+:command:`flashcp` but it doesn't always work: as it copies the new
+image over the top of the active root filesystem, surprise may ensue.
+Consider instead using a serial connection: you may need one anyway
+after trying flashcp if it corrupts the image.
+
+flashcp (not generally recommended)
+===================================
+
+Connect to the device and locate the "firmware" partition, which you
+can do with a combination of :command:`dmesg` output and the contents
+of :file:`/proc/mtd`
+
+.. code-block:: console
+
+   <5>[    0.469841] Creating 4 MTD partitions on "spi0.0":
+   <5>[    0.474837] 0x000000000000-0x000000040000 : "u-boot"
+   <5>[    0.480796] 0x000000040000-0x000000050000 : "u-boot-env"
+   <5>[    0.487056] 0x000000050000-0x000000060000 : "art"
+   <5>[    0.492753] 0x000000060000-0x000001000000 : "firmware"
+
+   # cat /proc/mtd
+   dev:    size   erasesize  name
+   mtd0: 00040000 00001000 "u-boot"
+   mtd1: 00010000 00001000 "u-boot-env"
+   mtd2: 00010000 00001000 "art"
+   mtd3: 00fa0000 00001000 "firmware"
+   mtd4: 002a0000 00001000 "kernel"
+   mtd5: 00d00000 00001000 "rootfs"
+
+Copy :file:`result/firmware.bin` to the device and now run (in this
+example)
+
+.. code-block:: console
+
+   flashcp -v firmware.bin /dev/mtd3
+
+
@@ -1,4 +1,4 @@
 Module options
 ##############

-.. include:: modules-generated.rst
+.. include:: modules-generated.inc.rst
@@ -10,4 +10,4 @@ different artefacts, or have different ways to get that artefact
 installed. The options available for a particular device are described in
 the section for that device.

-.. include:: outputs-generated.rst
+.. include:: outputs-generated.inc.rst
@@ -16,4 +16,4 @@
 (each [_ option (ipairs (sorted-options (yaml.load (io.read "*a"))))]
  (when (and (output? option) (not option.internal))
    (print (.. ".. _" (string.gsub option.name "%." "-") ":") "\n")
-    (print option.description)))
+    (print option.description "\n")))
@@ -71,7 +71,7 @@ stdin/stdout.
 You should now see Linux boot messages and after a few seconds be
 presented with a root shell prompt.  You can run commands to look at
 the filesystem, see what processes are running, view log messages (in
-:file:/run/uncaught-logs.current), etc. To kill the emulator, press ^P
+:file:/run/log/current), etc. To kill the emulator, press ^P
 (Control P) then c to enter the "QEMU Monitor", then type ``quit`` at
 the ``(qemu)`` prompt.

@@ -293,14 +293,17 @@ the hostname) and then run

 .. code-block:: console

-    nix-shell --run "liminix-rebuild root@address-of-the-device  -I liminix-config=./my-router.nix --arg device "import ./devices/gl-ar750""
+    nix-build  -I liminix-config=./my-router.nix \
+      --arg device "import ./devices/gl-ar750" \
+      -A outputs.systemConfiguration && \
+    result/install.sh root@address-of-the-device 

 (This requires the device to be network-accessible from your build
 machine, which for a test/demo system might involve a second network
 device in your build system - USB ethernet adapters are cheap - or
 a bit of messing around unplugging cables.)

-For more information about :code:`liminix-rebuild`, see the manual section :ref:`admin:Rebuilding the system`.
+For more information about in-place-updates, see the manual section :ref:`Rebuilding the system`.


 Final thoughts
@@ -52,7 +52,6 @@ in rec {
    dependencies = [ services.dhcpc ];
    name = "resolvconf";
    up = ''
-      . ${serviceFns}
      ( in_outputs ${name}
      for i in $(output ${services.dhcpc} dns); do
        echo "nameserver $i" > resolv.conf
@@ -93,7 +92,6 @@ in rec {
      secrets_file = oneshot rec {
        name = "rsync-secrets";
        up = ''
-          . ${serviceFns}
          (in_outputs ${name}
           echo  "backup:${secrets.rsync_secret}" > secrets)
        '';
@@ -137,7 +137,6 @@ in rec {
    dependencies = [ services.wan ];
    name = "resolvconf";
    up = ''
-      . ${serviceFns}
      ( in_outputs ${name}
       echo "nameserver $(output ${services.wan} ns1)" > resolv.conf
       echo "nameserver $(output ${services.wan} ns2)" >> resolv.conf
@@ -1,12 +1,14 @@
-{ config, pkgs, ... } :
+{ config, pkgs, lib, ... } :
 let
  svc = config.system.service;
+  inherit (pkgs.liminix.services) longrun oneshot ;

 in rec {
  imports = [
    ../modules/network
    ../modules/dnsmasq
    ../modules/ssh
+    ../modules/tls-certificate
  ];

  hostname = "hello";
@@ -17,8 +19,51 @@ in rec {
    family = "inet"; address ="10.3.0.1"; prefixLength = 16;
  };

+  filesystem =
+    let inherit (pkgs.pseudofile) file dir symlink;
+    in dir {
+      etc = dir {
+        hosts =  {
+          type = "f";
+          file = "127.0.0.1 localhost\n10.0.2.2 loaclhost.telent.net\n";
+          mode = "0444";
+        };
+      };
+    };
+
  services.sshd = svc.ssh.build { };

+  services.lan-address-for-secrets =
+    svc.network.address.build {
+      interface = config.hardware.networkInterfaces.lan;
+      family = "inet"; address ="10.0.2.15"; prefixLength = 24;
+    };
+
+  # services.client-cert = svc.tls-certificate.certifix-client.build {
+  #   caCertificate = builtins.readFile /var/lib/certifix/certs/ca.crt;
+  #   subject = "C=GB,ST=London,O=Telent,OU=devices,CN=${config.hostname}";
+  #   secret = builtins.readFile ../challengePassword;
+  #   serviceUrl = "https://loaclhost.telent.net:19613/sign";
+  # };
+
+  # logging.shipping = {
+  #   enable = true;
+  #   service = longrun {
+  #     name = "ship-logs";
+  #     dependencies = [ config.services.client-cert ];
+  #     run =
+  #       let path = lib.makeBinPath (with pkgs; [ s6-networking s6 ]);
+  #       in ''
+  #         PATH=${path}:$PATH \
+  #         CAFILE=${/var/lib/certifix/certs/ca.crt} \
+  #         KEYFILE=$(output_path ${services.client-cert} key) \
+  #         CERTFILE=$(output_path ${services.client-cert} cert) \
+  #         s6-tlsclient -k loaclhost.telent.net -h -y loaclhost.telent.net 19612 \
+  #         fdmove -c 1 7 cat
+  #       '';
+  #   };
+  # };
+
  users.root = {
    # the password is "secret". Use mkpasswd -m sha512crypt to
    # create this hashed password string
@@ -38,6 +83,6 @@ in rec {
    };

  defaultProfile.packages = with pkgs; [
-    figlet
+    figlet openssl
  ];
 }
@@ -1,141 +0,0 @@
-{
-  config,
-  pkgs,
-  lib,
-  ...
-}: let
-  secrets = import ./extneder-secrets.nix;
-  rsecrets = import ./rotuer-secrets.nix;
-
-  # https://support.aa.net.uk/Category:Incoming_L2TP says:
-  # "Please use the DNS name (l2tp.aa.net.uk) instead of hardcoding an
-  # IP address; IP addresses can and do change. If you have to use an
-  # IP, use 194.4.172.12, but do check the DNS for l2tp.aa.net.uk in
-  # case it changes."
-
-  # but (1) we don't want to use the wwan stick's dns as our main
-  # resolver: it's provided by some mobile ISP and they aren't
-  # necessarily the best at providing unfettered services without
-  # deciding to do something weird; (2) it's not simple to arrange
-  # that xl2tpd gets a different resolver than every other process;
-  # (3) there's no way to specify an lns address to xl2tpd at runtime
-  # except by rewriting its config file. So what we will do is lookup
-  # the lns hostname using the mobile ISP's dns server and then refuse
-  # to start l2tp unless the expected lns address is one of the
-  # addresses returned. I think this satisfies "do check the DNS"
-
-  lns = { hostname = "l2tp.aaisp.net.uk"; address = "194.4.172.12"; };
-
-  inherit (pkgs.liminix.services) oneshot target;
-  inherit (pkgs.pseudofile) dir symlink;
-  inherit (pkgs) serviceFns;
-  svc = config.system.service;
-in rec {
-  boot = {
-    tftp = {
-      serverip = "10.0.0.1";
-      ipaddr = "10.0.0.8";
-    };
-  };
-
-  imports = [
-    ../modules/cdc-ncm
-    ../modules/network
-    ../modules/vlan
-    ../modules/ssh
-    ../modules/usb.nix
-    ../modules/watchdog
-    ../modules/mount
-    ../modules/ppp
-  ];
-  hostname = "thing";
-
-  services.wwan = svc.wwan.build {
-    apn = "data.uk";
-    username = "user";
-    password = "one2one";
-    authType = "chap";
-  };
-
-  services.dhcpc = svc.network.dhcp.client.build {
-    interface = config.services.wwan;
-    dependencies = [ config.services.hostname ];
-  };
-
-  services.sshd = svc.ssh.build { };
-
-  services.resolvconf = oneshot rec {
-    dependencies = [ services.l2tp ];
-    name = "resolvconf";
-    up = ''
-      . ${serviceFns}
-       ( in_outputs ${name}
-        for i in ns1 ns2 ; do
-          ns=$(output ${services.l2tp} $i)
-          echo "nameserver $ns" >> resolv.conf
-        done
-       )
-    '';
-  };
-  filesystem = dir {
-    etc = dir {
-      "resolv.conf" = symlink "${services.resolvconf}/.outputs/resolv.conf";
-    };
-  };
-
-  services.lns-address = let
-    ns = "$(output_word ${services.dhcpc} dns 1)";
-    route-to-bootstrap-nameserver = svc.network.route.build {
-      via = "$(output ${services.dhcpc} router)";
-      target = ns;
-      dependencies = [services.dhcpc];
-    };
-  in oneshot rec {
-    name = "resolve-l2tp-server";
-    dependencies = [ services.dhcpc route-to-bootstrap-nameserver ];
-    up = ''
-      (in_outputs ${name}
-       DNSCACHEIP="${ns}" ${pkgs.s6-dns}/bin/s6-dnsip4 ${lns.hostname} \
-        > addresses
-      )
-    '';
-  };
-
-  services.l2tp =
-    let
-      check-address = oneshot rec {
-        name = "check-lns-address";
-        up = ''
-          grep -Fx ${lns.address} $(output_path ${services.lns-address} addresses)
-        '';
-        dependencies = [ services.lns-address ];
-      };
-      route = svc.network.route.build {
-        via = "$(output ${services.dhcpc} router)";
-        target = lns.address;
-        dependencies = [services.dhcpc check-address];
-      };
-    in svc.l2tp.build {
-      lns = lns.address;
-      ppp-options = [
-        "debug" "+ipv6" "noauth"
-        "name" rsecrets.l2tp.name
-        "connect-delay" "5000"
-        "password" rsecrets.l2tp.password
-      ];
-      dependencies = [config.services.lns-address route check-address];
-  };
-
-  services.defaultroute4 = svc.network.route.build {
-    via = "$(output ${services.l2tp} peer-address)";
-    target = "default";
-    dependencies = [services.l2tp];
-  };
-
-#  defaultProfile.packages = [ pkgs.go-l2tp ];
-
-  users.root = {
-    passwd = lib.mkForce secrets.root.passwd;
-    openssh.authorizedKeys.keys = secrets.root.keys;
-  };
-}
@@ -60,7 +60,6 @@ in rec {
    dependencies = [ services.dhcpc ];
    name = "resolvconf";
    up = ''
-      . ${serviceFns}
      ( in_outputs ${name}
      for i in $(output ${services.dhcpc} dns); do
        echo "nameserver $i" > resolv.conf
@@ -72,7 +71,6 @@ in rec {
  services.growfs = let name = "growfs"; in oneshot {
    inherit name;
    up = ''
-      . ${serviceFns}
      device=$(grep /persist /proc/1/mountinfo | cut -f9 -d' ')
      ${pkgs.e2fsprogs}/bin/resize2fs $device
    '';
@@ -1,10 +1,8 @@
-# This is not part of Liminix per se. This is my "scratchpad"
-# configuration for the device I'm testing with.
-#
-# Parts of it do do things that Liminix eventually needs to do, but
-# don't look in here for solutions - just for identifying the
-# problems.
-
+# This is an example that uses the "gateway" profile to create a
+# "typical home wireless router" configuration suitable for a Gl.inet
+# gl-ar750 router. It should be fairly simple to edit it for other
+# devices: mostly you will need to attend to the number of wlan and lan
+# interfaces

 { config, pkgs, lib, modulesPath, ... } :
 let
@@ -18,11 +16,11 @@ let
    inherit (secrets) wpa_passphrase;
    wmm_enabled = 1;
  };
-
+  inherit (pkgs.liminix.services) longrun;
 in rec {
  boot = {
    tftp = {
-      freeSpaceBytes = 3 * 1024 * 1024;
+      freeSpaceBytes = 2 * 1024 * 1024;
      serverip = "10.0.0.1";
      ipaddr = "10.0.0.8";
    };
@@ -30,21 +28,34 @@ in rec {

  imports = [
    "${modulesPath}/profiles/gateway.nix"
-    "${modulesPath}/schnapps"
-    "${modulesPath}/outputs/btrfs.nix"
-    "${modulesPath}/outputs/extlinux.nix"
+    "${modulesPath}/tls-certificate"
  ];
+
  hostname = "rotuer";
-  rootfsType = "btrfs";
-  rootOptions = "subvol=@";
-  boot.loader.extlinux.enable = true;
+#  rootfsType = "jffs2";
+
+  filesystem =
+    let inherit (pkgs.pseudofile) file dir symlink;
+    in dir {
+      mnt = dir {};
+      etc = dir {
+        hosts = {
+          type = "f";
+          file = "127.0.0.1 localhost\n10.0.0.1 loaclhost.telent.net\n";
+          mode = "0444";
+        };
+      };
+    };

  profile.gateway = {
    lan = {
      interfaces =  with config.hardware.networkInterfaces;
        [
+          # EDIT: these are the interfaces exposed by the gl.inet gl-ar750:
+          # if your device has more or differently named lan interfaces,
+          # specify them here
          wlan wlan5
-          lan0 lan1 lan2 lan3 lan4
+          lan
        ];
      inherit (secrets.lan) prefix;
      address = {
@@ -58,9 +69,17 @@ in rec {
      };
    };
    wan = {
-      interface = config.hardware.networkInterfaces.wan;
-      username = secrets.l2tp.name;
-      password = secrets.l2tp.password;
+      # wan interface depends on your upstream - could be dhcp, static
+      # ethernet, a pppoe, ppp over serial, a complicated bonded
+      # failover ... who knows what else?
+      interface = svc.pppoe.build {
+        interface = config.hardware.networkInterfaces.wan;
+        username = secrets.l2tp.name;
+        password = secrets.l2tp.password;
+      };
+      # once the wan has ipv4 connnectivity, should we run dhcp6
+      # client to potentially get an address range ("prefix
+      # delegation")
      dhcp6.enable = true;
    };
    firewall = {
@@ -68,6 +87,10 @@ in rec {
      rules = secrets.firewallRules;
    };
    wireless.networks = {
+      # EDIT: if you have more or fewer wireless radios, here is where
+      # you need to say so.  hostapd tuning is hardware-specific and
+      # left as an exercise for the reader :-).
+
      "${secrets.ssid}" = {
        interface = config.hardware.networkInterfaces.wlan;
        hw_mode = "g";
@@ -99,9 +122,11 @@ in rec {
  defaultProfile.packages = with pkgs; [
    min-collect-garbage
    nftables
-    strace
-    tcpdump
+    # strace
+    # tcpdump
    s6
+    dtc
+    # certifix-client
  ];

  programs.busybox = {
@@ -112,4 +137,47 @@ in rec {
      FEATURE_FANCY_TAIL = "y";
    };
  };
+
+  services.wan-address-for-secrets =
+    let
+      interface = config.hardware.networkInterfaces.wan;
+      addr =
+        svc.network.address.build {
+          inherit interface;
+          family = "inet"; address ="10.0.0.10"; prefixLength = 24;
+        };
+    in svc.network.route.build {
+      target = "10.0.0.1";
+      inherit interface;
+      via = "10.0.0.10";
+      metric = 1;
+      dependencies = [ addr ];
+    };
+
+  # services.client-cert = svc.tls-certificate.certifix-client.build {
+  #   caCertificate = builtins.readFile /var/lib/certifix/certs/ca.crt;
+  #   subject = "C=GB,ST=London,O=Telent,OU=devices,CN=${config.hostname}";
+  #   secret = builtins.readFile ../challengePassword;
+  #   serviceUrl = "https://loaclhost.telent.net:19613/sign";
+  #   dependencies = [ services.wan-address-for-secrets ];
+  # };
+
+  logging.pstore = true;
+  logging.shipping = {
+    enable = false;
+    service = longrun {
+      name = "ship-logs";
+      dependencies = [ config.services.client-cert ];
+      run =
+        let path = lib.makeBinPath (with pkgs; [ s6-networking s6 ]);
+        in ''
+          PATH=${path}:$PATH \
+          CAFILE=${/var/lib/certifix/certs/ca.crt} \
+          KEYFILE=$(output_path ${services.client-cert} key) \
+          CERTFILE=$(output_path ${services.client-cert} cert) \
+          s6-tlsclient -k loaclhost.telent.net -h -y loaclhost.telent.net 19612 \
+          fdmove -c 1 7 cat
+        '';
+    };
+  };
 }
@@ -0,0 +1,211 @@
+# A demonstration config for a home/soho router with PPPoE upstream
+# and fallback to an L2TP tunnel over a USB WWAN device
+
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}: let
+  secrets = import ./extneder-secrets.nix;
+  rsecrets = import ./rotuer-secrets.nix;
+
+  # https://support.aa.net.uk/Category:Incoming_L2TP says:
+  # "Please use the DNS name (l2tp.aa.net.uk) instead of hardcoding an
+  # IP address; IP addresses can and do change. If you have to use an
+  # IP, use 194.4.172.12, but do check the DNS for l2tp.aa.net.uk in
+  # case it changes."
+
+  # but (1) we don't want to use the wwan stick's dns as our main
+  # resolver: it's provided by some mobile ISP and they aren't
+  # necessarily the best at providing unfettered services without
+  # deciding to do something weird; (2) it's not simple to arrange
+  # that xl2tpd gets a different resolver than every other process;
+  # (3) there's no way to specify an lns address to xl2tpd at runtime
+  # except by rewriting its config file. So what we will do is lookup
+  # the lns hostname using the mobile ISP's dns server and then refuse
+  # to start l2tp unless the expected lns address is one of the
+  # addresses returned. I think this satisfies "do check the DNS"
+
+  lns = { hostname = "l2tp.aaisp.net.uk"; address = "194.4.172.12"; };
+
+  inherit (pkgs.liminix.services) oneshot longrun target;
+  inherit (pkgs.liminix) outputRef;
+  inherit (pkgs.pseudofile) dir symlink;
+  inherit (pkgs) serviceFns;
+  svc = config.system.service;
+  wirelessConfig = {
+    country_code = "GB";
+    inherit (rsecrets) wpa_passphrase;
+    wmm_enabled = 1;
+  };
+in rec {
+  boot = {
+    tftp = {
+      serverip = "10.0.0.1";
+      ipaddr = "10.0.0.8";
+    };
+  };
+
+  imports = [
+    ../modules/wwan
+    ../modules/network
+    ../modules/ssh
+    ../modules/usb.nix
+    ../modules/ppp
+    ../modules/round-robin
+    ../modules/health-check
+    ../modules/secrets
+    ../modules/profiles/gateway.nix
+  ];
+  hostname = "thing";
+
+  services.wan-address-for-secrets = svc.network.address.build {
+    interface = config.hardware.networkInterfaces.wan;
+    family = "inet"; address ="10.0.0.10"; prefixLength = 24;
+  };
+
+  services.secrets = svc.secrets.outboard.build {
+    name = "secret-service";
+    url = "http://10.0.0.1/liminix/examples/real-secrets.json";
+    username = "demo";
+    password = "demo";
+    interval = 5;
+    dependencies = [ services.wan-address-for-secrets ];
+  };
+
+  services.wwan = svc.wwan.huawei-e3372.build {
+    apn = "data.uk";
+    username = "user";
+    password = "one2one";
+    authType = "chap";
+  };
+
+  profile.gateway = {
+    lan = {
+      interfaces =  with config.hardware.networkInterfaces;
+        [
+          # EDIT: these are the interfaces exposed by the gl.inet gl-ar750:
+          # if your device has more or differently named lan interfaces,
+          # specify them here
+          wlan wlan5
+          lan
+        ];
+      inherit (rsecrets.lan) prefix;
+      address = {
+        family = "inet"; address ="${rsecrets.lan.prefix}.1"; prefixLength = 24;
+      };
+      dhcp = {
+        start = 10;
+        end = 240;
+        hosts = { } // lib.optionalAttrs (builtins.pathExists ./static-leases.nix) (import ./static-leases.nix);
+        localDomain = "lan";
+      };
+    };
+    wan =
+      let
+        secret = outputRef config.services.secrets;
+        username = secret "ppp/username";
+        password = secret "ppp/password";
+      in {
+        interface =
+          let
+            pppoe = svc.pppoe.build {
+              interface = config.hardware.networkInterfaces.wan;
+              debug = true;
+              inherit username password;
+            };
+
+            l2tp =
+              let
+                check-address = oneshot rec {
+                  name = "check-lns-address";
+                  up = "grep -Fx ${lns.address} $(output_path ${services.lns-address} addresses)";
+                  dependencies = [ services.lns-address ];
+                };
+                route = svc.network.route.build {
+                  via = "$(output ${services.bootstrap-dhcpc} router)";
+                  target = lns.address;
+                  dependencies = [services.bootstrap-dhcpc check-address];
+                };
+                l2tpd= svc.l2tp.build {
+                  lns = lns.address;
+                  inherit username password;
+                  dependencies = [config.services.lns-address route check-address];
+                };
+              in
+                svc.health-check.build {
+                  service = l2tpd;
+                  threshold = 3;
+                  interval = 2;
+                  healthCheck = pkgs.writeAshScript "ping-check" {} "ping 1.1.1.1";
+                };
+          in svc.round-robin.build {
+            name = "wan";
+            services = [
+              pppoe
+              l2tp
+            ];
+          };
+        dhcp6.enable = true;
+      };
+
+    wireless.networks = {
+      "${rsecrets.ssid}" = {
+        interface = config.hardware.networkInterfaces.wlan;
+        hw_mode = "g";
+        channel = "6";
+        ieee80211n = 1;
+      } // wirelessConfig // {
+        wpa_passphrase = outputRef config.services.secrets "wpa_passphrase";
+      };
+
+      "${rsecrets.ssid}5" = rec {
+        interface = config.hardware.networkInterfaces.wlan5;
+        hw_mode = "a";
+        channel = 36;
+        ht_capab = "[HT40+]";
+        vht_oper_chwidth = 1;
+        vht_oper_centr_freq_seg0_idx = channel + 6;
+        ieee80211n = 1;
+        ieee80211ac = 1;
+      } // wirelessConfig // {
+        wpa_passphrase = outputRef config.services.secrets "wpa_passphrase";
+      };
+    };
+  };
+
+  services.bootstrap-dhcpc = svc.network.dhcp.client.build {
+    interface = config.services.wwan;
+    dependencies = [ config.services.hostname ];
+  };
+
+  services.sshd = svc.ssh.build {
+    authorizedKeys = outputRef config.services.secrets "ssh/authorizedKeys";
+  };
+
+  services.lns-address = let
+    ns = "$(output_word ${services.bootstrap-dhcpc} dns 1)";
+    route-to-bootstrap-nameserver = svc.network.route.build {
+      via = "$(output ${services.bootstrap-dhcpc} router)";
+      target = ns;
+      dependencies = [services.bootstrap-dhcpc];
+    };
+  in oneshot rec {
+    name = "resolve-l2tp-server";
+    dependencies = [ services.bootstrap-dhcpc route-to-bootstrap-nameserver ];
+    up = ''
+      (in_outputs ${name}
+       DNSCACHEIP="${ns}" ${pkgs.s6-dns}/bin/s6-dnsip4 ${lns.hostname} \
+        > addresses
+      )
+    '';
+  };
+
+  users.root = rsecrets.root;
+
+  programs.busybox.options = {
+    FEATURE_FANCY_TAIL = "y";
+  };
+
+}
@@ -0,0 +1,19 @@
+{
+    "wpa_passphrase": "you bring light in",
+    "ssid": "liminix",
+    "l2tp": {
+	"name": "abcde@a.1",
+	"password": "NotMyIspPassword"
+    },
+    "root": {
+	"passwd": "$6$6pt0mpbgcB7kC2RJ$kSBoCYGyi1.qxt7dqmexLj1l8E6oTZJZmfGyJSsMYMW.jlsETxdgQSdv6ptOYDM7DHAwf6vLG0pz3UD31XBfC1",
+	"openssh": {
+	    "authorizedKeys": {
+		"keys": [ ]
+	    }
+	}
+    },
+    "lan": {
+	"prefix": "10.8.0"
+    }
+}
@@ -22,7 +22,7 @@ in
    };
  };
  config.system.service.bridge = {
-    primary = liminix.callService ./primary.nix {
+    primary = config.system.callService ./primary.nix {
      ifname = mkOption {
        type = types.str;
        description = "bridge interface name to create";
@@ -1,31 +0,0 @@
-{ config, pkgs, lib, ... }:
-let
-  inherit (pkgs) liminix;
-  inherit (lib) mkOption types;
-in {
-  imports = [
-    ../service-trigger
-  ];
-
-  options = {
-    system.service.wwan = mkOption {
-      type = liminix.lib.types.serviceDefn;
-    };
-  };
-  config = {
-    kernel.config = {
-      USB_NET_HUAWEI_CDC_NCM = "y";
-      USB_USBNET = "y";
-      USB_SERIAL = "y";
-      USB_SERIAL_OPTION = "y";
-    };
-
-    # https://www.0xf8.org/2017/01/flashing-a-huawei-e3372h-4g-lte-stick-from-hilink-to-stick-mode/
-    system.service.wwan = config.system.callService ./wwan.nix {
-      apn = mkOption { type = types.str; };
-      username = mkOption { type = types.str; };
-      password = mkOption { type = types.str; };
-      authType = mkOption { type = types.enum [ "pap" "chap" ]; };
-    };
-  };
-}
@@ -27,6 +27,6 @@
        dir (svc.open state-directory)]
    (accumulate [addresses []
                 v (dir:events)]
-      (update-prefixes lan-device addresses (v:output "prefix") system))))
+      (update-prefixes lan-device addresses (or (v:output "prefix") []) system))))

 { : changes : run }
@@ -27,6 +27,6 @@
        dir (svc.open state-directory)]
    (accumulate [addresses []
                 v (dir:events)]
-      (update-addresses wan-device addresses (v:output "address") system))))
+      (update-addresses wan-device addresses (or (v:output "address") []) system))))

 { : update-addresses : deletions : run }
@@ -3,7 +3,6 @@
 , linotify
 , anoia
 , lualinux
-, lua
 }:
 writeFennel "acquire-wan-address" {
  packages = [ linotify anoia lualinux ];
@@ -23,13 +23,13 @@ in
    };
  };
  config.system.service.dhcp6c = {
-    client = liminix.callService ./client.nix {
+    client = config.system.callService ./client.nix {
      interface = mkOption {
        type = liminix.lib.types.interface;
        description = "interface (usually WAN) to query for DHCP6";
      };
    };
-    address = liminix.callService ./address.nix {
+    address = config.system.callService ./address.nix {
      client = mkOption {
        type = types.anything; # liminix.lib.types.service;
      };
@@ -38,7 +38,7 @@ in
        description = "interface to assign the address to";
      };
    };
-    prefix = liminix.callService ./prefix.nix {
+    prefix = config.system.callService ./prefix.nix {
      client = mkOption {
        type = types.anything; # liminix.lib.types.service;
      };
@@ -16,7 +16,7 @@ in {
    };
  };
  config = {
-    system.service.dnsmasq = liminix.callService ./service.nix {
+    system.service.dnsmasq = config.system.callService ./service.nix {
      user = mkOption {
        type = types.str;
        default = "dnsmasq";
@@ -26,7 +26,6 @@ longrun {
  inherit name;
  dependencies = [ interface ];
  run = ''
-    . ${serviceFns}
    ${dnsmasq}/bin/dnsmasq \
    --user=${user} \
    --domain=${domain} \
@@ -90,7 +90,7 @@ in {
      # accept inbound from the WAN
      (if allow-incoming
       then accept "oifname \"int\" iifname \"ppp0\""
-       else "oifname \"int\" iifname \"ppp0\" jump incoming-allowed-ip6"
+       else "iifname \"ppp0\" jump incoming-allowed-ip6"
      )
      # allow all outbound and any inbound that's part of a
      # recognised (outbound-initiated) flow
@@ -210,7 +210,7 @@ in {
      "icmp type { echo-request, echo-reply } accept"
      "iifname int jump input-ip4-lan"
      "iifname ppp0 jump input-ip4-wan"
-      "oifname \"int\" iifname \"ppp0\" jump incoming-allowed-ip4"
+      "iifname ppp0 jump incoming-allowed-ip4"
      "ct state established,related accept"
      "log prefix \"DENIED CHAIN=input-ip4 \""
    ];
@@ -54,7 +54,7 @@ in
  };
  config = {
    system.service.firewall =
-      let svc = liminix.callService ./service.nix  {
+      let svc = config.system.callService ./service.nix  {
            extraRules = mkOption {
              type = types.attrsOf types.attrs;
              description = "firewall ruleset";
@@ -0,0 +1,43 @@
+## Health check
+##
+## Runs a service and a separate periodic health process. When the
+## health check starts failing over a period of time, kill the service.
+## (Usually that means the supervisor will restart it, but you can
+## have other behaviours by e.g. combining this service with a round-robin
+## for failover)
+
+
+{ lib, pkgs, config, ...}:
+let
+  inherit (lib) mkOption types;
+  inherit (pkgs) liminix;
+#  inherit (pkgs.liminix.services) longrun;
+in {
+  options = {
+    system.service.health-check = mkOption {
+      description = "run a service while periodically checking it is healthy";
+      type = liminix.lib.types.serviceDefn;
+    };
+  };
+  config.system.service.health-check = config.system.callService ./service.nix {
+    service = mkOption {
+      type = liminix.lib.types.service;
+    };
+    interval =  mkOption {
+      description = "interval between checks, in seconds";
+      type = types.int;
+      default = 10;
+      example = 10;
+    };
+    threshold =  mkOption {
+      description = "number of consecutive failures required for the service to be kicked";
+      type = types.int;
+      example = 3;
+    };
+    healthCheck = mkOption {
+      description = "health check command or script. Expected to exit 0 if the service is healthy or any other exit status otherwise";
+      type = types.path;
+    };
+  };
+  config.programs.busybox.applets = ["expr"];
+}
@@ -0,0 +1,37 @@
+{
+  liminix, lib, lim, s6
+}:
+{ service, interval, threshold, healthCheck } :
+let
+  inherit (liminix.services) oneshot longrun;
+  inherit (builtins) toString;
+  inherit (service) name;
+  checker = let name' = "check-${name}"; in longrun {
+    name = name';
+    run = ''
+      fails=0
+      echo waiting for /run/service/${name}
+      ${s6}/bin/s6-svwait -U /run/service/${name} || exit
+      while sleep ${toString interval} ; do
+        ${healthCheck}
+        if test $? -gt 0; then
+          fails=$(expr $fails + 1)
+        else
+          fails=0
+        fi
+        echo fails $fails/${toString threshold} for ${name}
+        if test "$fails" -gt "${toString threshold}" ; then
+          echo time to die
+          ${s6}/bin/s6-svc -r /run/service/${name}
+          echo bounced
+          fails=0
+          echo waiting for /run/service/${name}
+          ${s6}/bin/s6-svwait -U /run/service/${name}
+        fi
+      done
+    '';
+  };
+in service.overrideAttrs(o: {
+  buildInputs =  (lim.orEmpty o.buildInputs) ++ [ checker ];
+  dependencies = (lim.orEmpty o.dependencies) ++ [ checker ];
+})
@@ -16,13 +16,14 @@ let
  inherit (lib) mkOption types;
  inherit (pkgs) liminix;
 in {
+  imports = [ ../secrets ];
  options = {
    system.service.hostapd = mkOption {
      type = liminix.lib.types.serviceDefn;
    };
  };
  config = {
-    system.service.hostapd = liminix.callService ./service.nix {
+    system.service.hostapd = config.system.callService ./service.nix {
      interface = mkOption {
        type = liminix.lib.types.service;
      };
@@ -1,13 +1,16 @@
 {
  liminix
+, svc
 , hostapd
+, output-template
 , writeText
 , lib
 }:
 { interface, params} :
 let
  inherit (liminix.services) longrun;
-  inherit (lib) concatStringsSep mapAttrsToList;
+  inherit (lib) concatStringsSep mapAttrsToList unique ;
+  inherit (builtins) map filter attrValues length head typeOf;

  # This is not a friendly interface to configuring a wireless AP: it
  # just passes everything straight through to the hostapd config.
@@ -20,18 +23,35 @@ let
    driver = "nl80211";
    logger_syslog = "-1";
    logger_syslog_level = 1;
-    ctrl_interface = "/run/hostapd";
+    ctrl_interface = "/run/${name}";
    ctrl_interface_group = 0;
  };
+  attrs = defaults // params ;
+  literal_or_output = o: ({
+    string = builtins.toJSON;
+    int = builtins.toJSON;
+    lambda = (o: "output(${builtins.toJSON (o "service")}, ${builtins.toJSON (o "path")})");
+  }.${builtins.typeOf o}) o;

-  conf = writeText "hostapd.conf"
-    (concatStringsSep
-      "\n"
-      (mapAttrsToList
-        (name: value: "${name}=${toString value}")
-        (defaults // params)));
-in longrun {
-  inherit name;
-  dependencies = [ interface ];
-  run = "${hostapd}/bin/hostapd -i $(output ${interface} ifname)  -P /run/${name}.pid -S ${conf}";
+  conf =
+    (writeText "hostapd.conf.in"
+      ((concatStringsSep
+        "\n"
+        (mapAttrsToList
+          (n : v : "${n}={{ ${literal_or_output v} }}")
+          attrs)) + "\n"));
+  service = longrun {
+    inherit name;
+    dependencies = [ interface ];
+    run = ''
+      mkdir -p /run/${name}
+      chmod 0700 /run/${name}
+      ${output-template}/bin/output-template '{{' '}}' < ${conf} > /run/${name}/hostapd.conf
+      exec ${hostapd}/bin/hostapd -i $(output ${interface} ifname) -P /run/${name}/hostapd.pid -S /run/${name}/hostapd.conf
+    '';
+  };
+  watch = filter (f: typeOf f == "lambda") (attrValues attrs);
+in svc.secrets.subscriber.build {
+  inherit service watch;
+  action = "restart-all";
 }
@@ -91,9 +91,8 @@ in {
        IKCONFIG_PROC = "y";
        PROC_FS = "y";

-        KEXEC = "y";
        MODULES = if modular then "y" else "n";
-        MODULE_SIG = if modular then "y" else "n";
+        MODULE_UNLOAD = if modular then "y" else "n";
        DEBUG_FS = "y";

        # basic networking protocols
@@ -0,0 +1,15 @@
+{ config, lib, pkgs, ... }:
+let
+  inherit (pkgs.liminix.services) longrun;
+in {
+  config.services.klogd = longrun {
+    name = "klogd";
+    run = ''
+      echo "1 2 1 8"  > /proc/sys/kernel/printk
+      cat /proc/kmsg
+    '';
+    finish = ''
+      echo "8 4 1 8"  > /proc/sys/kernel/printk
+    '';
+  };
+}
@@ -11,7 +11,9 @@ in {
      devout = longrun {
        name = "devout";
        notification-fd = 10;
+        timeout-up = 60 * 1000;
        run = "exec ${pkgs.devout}/bin/devout /run/devout.sock 4";
+        dependencies = [ mdevd ];
      };
      coldplug = oneshot {
        name = "coldplug";
@@ -14,7 +14,7 @@ in {
      type = liminix.lib.types.serviceDefn;
    };
  };
-  imports = [ ../mdevd.nix ../service-trigger ];
+  imports = [ ../mdevd.nix ../uevent-rule ];
  config.system.service.mount =
    let svc = config.system.callService ./service.nix {
          partlabel = mkOption {
@@ -11,7 +11,6 @@ let
  # prefixes, or the same but different protocols
  name = "${interface.name}.a.${address}";
  up = ''
-    . ${serviceFns}
    dev=$(output ${interface} ifname)
    ip address add ${address}/${toString prefixLength} dev $dev
    (in_outputs ${name}
@@ -64,7 +64,7 @@ in {
    services.loopback = config.hardware.networkInterfaces.lo;

    system.service.network = {
-      link = liminix.callService ./link.nix {
+      link = config.system.callService ./link.nix {
        ifname = mkOption {
          type = types.str;
          example = "eth0";
@@ -89,7 +89,7 @@ in {
          example = 1480;
        };
      };
-      address = liminix.callService ./address.nix {
+      address = config.system.callService ./address.nix {
        interface = mkOption {
          type = liminix.lib.types.service;
        };
@@ -104,7 +104,7 @@ in {
        };
      };

-      route = liminix.callService ./route.nix {
+      route = config.system.callService ./route.nix {
        interface = mkOption {
          type = types.nullOr liminix.lib.types.interface;
          default = null;
@@ -125,7 +125,7 @@ in {
        };
      };

-      forward = liminix.callService ./forward.nix {
+      forward = config.system.callService ./forward.nix {
        enableIPv4 = mkOption {
          type = types.bool;
          default = true;
@@ -136,7 +136,7 @@ in {
        };
      };

-      dhcp.client = liminix.callService ./dhcpc.nix {
+      dhcp.client = config.system.callService ./dhcpc.nix {
        interface = mkOption {
          type = liminix.lib.types.service;
        };
@@ -18,7 +18,7 @@ in {
    };
  };
  config = {
-    system.service.ntp = liminix.callService ./service.nix {
+    system.service.ntp = config.system.callService ./service.nix {
      user = mkOption {
        type = types.str;
        default = "ntp";
@@ -6,6 +6,7 @@
 }:
 params:
 let
+  name = "ntp"; # bad name, needs to be unique
  inherit (liminix.services) longrun;
  inherit (lib) concatStringsSep mapAttrsToList;
  configFile = p:
@@ -23,11 +24,15 @@ let
    ++ (lib.optional (p.bindaddress != null) "bindaddress ${p.bindaddress}")
    ++ (lib.optional (p.binddevice != null) "binddevice ${p.binddevice}")
    ++ (lib.optional (p.dumpdir != null) "dumpdir ${p.dumpdir}")
+    ++ [
+      "bindcmdaddress /" # disable unix socket
+      "pidfile /run/${name}.pid"
+    ]
    ++ [p.extraConfig];

  config = writeText "chrony.conf"
    (concatStringsSep "\n" (configFile params));
 in longrun {
-  name = "ntp"; # bad name, needs to be unique
+  inherit name;
  run = "${chrony}/bin/chronyd -f ${config} -d";
 }
@@ -12,6 +12,9 @@ in {
    kexecboot = mkOption {
      type = types.package;
      description = ''
+        kexecboot
+        *********
+
        Directory containing files needed for kexec booting.
        Can be copied onto the target device using ssh or similar
      '';
@@ -0,0 +1,92 @@
+{ writeAshScript, liminix, svc, lib, serviceFns, output-template }:
+{
+  command,
+  name,
+  debug
+, username,
+  password,
+  lcpEcho,
+  ppp-options,
+  dependencies ? []
+} :
+let
+  inherit (lib) optional optionals escapeShellArgs concatStringsSep;
+  inherit (liminix.services) longrun;
+  inherit (builtins) toJSON toString typeOf;
+
+  ip-up = writeAshScript "ip-up" {} ''
+    exec >&5 2>&5
+    . ${serviceFns} 
+    in_outputs ${name}
+    echo $1 > ifname
+    echo $2 > tty
+    echo $3 > speed
+    echo $4 > address
+    echo $5 > peer-address
+    set +o nounset
+    set +o errexit
+    if test -n "''${DNS1}" ;then echo ''${DNS1} > ns1 ; fi
+    if test -n "''${DNS2}" ;then echo ''${DNS2} > ns2 ; fi
+    set +o pipefail
+    ls -l /proc/self/fd
+    echo >/proc/self/fd/10
+  '';
+  ip6-up = writeAshScript "ip6-up" {} ''
+    exec >&5 2>&5
+    . ${serviceFns} 
+    in_outputs ${name}
+    set +o nounset
+    set +o errexit
+    echo $4 > ipv6-address
+    echo $5 > ipv6-peer-address
+  '';
+  literal_or_output =
+    let v = o: ({
+          string = toJSON;
+          int = toJSON;
+          lambda = (o: "output(${toJSON (o "service")}, ${toJSON (o "path")})");
+        }.${typeOf o}) o;
+    in  o: "{{ ${v o} }}";
+
+  ppp-options' =
+    ["+ipv6" "noauth"]
+    ++ optional debug "debug"
+    ++ optionals (username != null) ["name" (literal_or_output username)]
+    ++ optionals (password != null) ["password" (literal_or_output password)]
+    ++ optional lcpEcho.adaptive "lcp-echo-adaptive"
+    ++ optionals (lcpEcho.interval != null)
+      ["lcp-echo-interval" (toString lcpEcho.interval)]
+    ++ optionals (lcpEcho.failure != null)
+      ["lcp-echo-failure" (toString lcpEcho.failure)]
+    ++ ppp-options
+    ++ ["ip-up-script" ip-up
+        "ipv6-up-script" ip6-up
+        "ipparam" name
+        "nodetach"
+        # usepeerdns requests DNS servers from peer (which is good),
+        # then attempts to write them to /nix/store/xxxx/ppp/resolv.conf
+        # which causes an unsightly but inconsequential error message
+        "usepeerdns"
+        "nodefaultroute"
+        "logfd" "2"
+       ];
+  service = longrun {
+    inherit name;
+    run = ''
+      mkdir -p /run/${name}
+      chmod 0700 /run/${name}
+      in_outputs ${name}
+      echo ${escapeShellArgs ppp-options'} | ${output-template}/bin/output-template '{{' '}}' > /run/${name}/ppp-options
+      exec fdmove -c 5 2 \
+      ${command}
+    '';
+    notification-fd = 10;
+    timeout-up = if lcpEcho.failure != null
+                 then (10 + lcpEcho.failure * lcpEcho.interval) * 1000
+                 else 60 * 1000;
+    inherit dependencies;
+  };
+in svc.secrets.subscriber.build {
+  watch = lib.filter (n: typeOf n=="lambda") [ username password ];
+  inherit service;
+}
@@ -1,18 +1,31 @@
 ## PPP
 ## ===
 ##
-## A PPPoE (PPP over Ethernet) configuration to address the case where
-## your Liminix device is connected to an upstream network using
+## ``ppoe`` (PPP over Ethernet) provides a service to address the case
+## where your Liminix device is connected to an upstream network using
 ## PPPoE. This is typical for UK broadband connections where the
 ## physical connection is made by OpenReach ("Fibre To The X") and
-## common in some other localities as well: ask your ISP if this is
+## common in some other localities as well: check with your ISP if this is
 ## you.
+##
+## ``l2tp`` (Layer 2 Tunelling Protocol) provides a service that
+## tunnels PPP over the Internet.  This may be used by some ISPs in
+## conjunction with a DHCP uplink, or other more creative forms of
+## network connection
+

 { lib, pkgs, config, ...}:
 let
  inherit (lib) mkOption types;
  inherit (pkgs) liminix;
+  mkStringOption =
+    description: mkOption {
+      type = types.nullOr types.str;
+      default = null;
+      inherit description;
+    };
 in {
+  imports = [ ../secrets ];
  options = {
    system.service.pppoe = mkOption {
      type = liminix.lib.types.serviceDefn;
@@ -22,23 +35,89 @@ in {
    };
  };
  config = {
-    system.service.pppoe = pkgs.liminix.callService ./pppoe.nix {
+    system.service.pppoe = config.system.callService ./pppoe.nix {
      interface = mkOption {
        type = liminix.lib.types.service;
        description = "ethernet interface to run PPPoE over";
      };
+      username = mkOption {
+        type = types.nullOr (liminix.lib.types.replacable types.str);
+        default = null;
+        description = "username";
+      };
+      password = mkOption {
+        type = types.nullOr (liminix.lib.types.replacable types.str);
+        default = null;
+        description = "password";
+      };
+      lcpEcho = {
+        adaptive = mkOption {
+          description = "send LCP echo-request frames only if no traffic was received from the peer since the last echo-request was sent";
+          type = types.bool;
+          default = true;
+        };
+        interval = mkOption {
+          type = types.nullOr types.int;
+          default = 3;
+          description = "send an LCP echo-request frame to the peer every n seconds";
+        };
+        failure =  mkOption {
+          type = types.nullOr types.int;
+          default = 3;
+          description = "terminate connection if n LCP echo-requests are sent without receiving a valid LCP echo-reply";
+        };
+      };
+      debug = mkOption {
+        description = "log the contents of all control packets sent or received";
+        default = false;
+        type = types.bool;
+      };
      ppp-options = mkOption {
        type = types.listOf types.str;
        description = "options supplied on ppp command line";
+        default = [];
      };
    };
-    system.service.l2tp = pkgs.liminix.callService ./l2tp.nix {
+    system.service.l2tp = config.system.callService ./l2tp.nix {
      lns = mkOption {
        type = types.str;
        description = "hostname or address of the L2TP network server";
      };
+      username = mkOption {
+        type = types.nullOr (liminix.lib.types.replacable types.str);
+        default = null;
+        description = "username";
+      };
+      password = mkOption {
+        type = types.nullOr (liminix.lib.types.replacable types.str);
+        default = null;
+        description = "password";
+      };
+      lcpEcho = {
+        adaptive = mkOption {
+          description = "send LCP echo-request frames only if no traffic was received from the peer since the last echo-request was sent";
+          type = types.bool;
+          default = true;
+        };
+        interval = mkOption {
+          type = types.nullOr types.int;
+          default = 3;
+          description = "send an LCP echo-request frame to the peer every n seconds";
+        };
+        failure =  mkOption {
+          type = types.nullOr types.int;
+          default = 3;
+          description = "terminate connection if n LCP echo-requests are sent without receiving a valid LCP echo-reply";
+        };
+      };
+      debug = mkOption {
+        description = "log the contents of all control packets sent or received";
+        default = false;
+        type = types.bool;
+      };
      ppp-options = mkOption {
        type = types.listOf types.str;
+        default = [];
        description = "options supplied on ppp command line";
      };
    };
@@ -1,59 +1,40 @@
 {
-  liminix
-, writeAshScript
-, writeText
-, serviceFns
-, xl2tpd
+  lib,
+  liminix,
+  output-template,
+  serviceFns,
+  svc,
+  writeAshScript,
+  writeText,
+  xl2tpd,
+  callPackage
 } :
-{ lns, ppp-options }:
+{ lns,
+  ppp-options,
+  lcpEcho,
+  username,
+  password,
+  debug
+}:
 let
-  inherit (liminix.services) longrun;
  name = "${lns}.l2tp";
-  ip-up = writeAshScript "ip-up" {} ''
-    . ${serviceFns} 
-    (in_outputs ${name}
-     echo $1 > ifname
-     echo $2 > tty
-     echo $3 > speed
-     echo $4 > address
-     echo $5 > peer-address
-     echo $DNS1 > ns1
-     echo $DNS2 > ns2
-    )
-    echo >/proc/self/fd/10
-  '';
-  ip6-up = writeAshScript "ip6-up" {} ''
-    . ${serviceFns} 
-    (in_outputs ${name}
-     echo $4 > ipv6-address
-     echo $5 > ipv6-peer-address
-    )
-    echo >/proc/self/fd/10
-  '';
-  ppp-options' = ppp-options ++ [
-    "ip-up-script" ip-up
-    "ipv6-up-script" ip6-up
-    "ipparam" name
-    "nodetach"
-    "usepeerdns"
-    "logfd" "2"
-  ];
+  common = callPackage ./common.nix { inherit svc; };
+
  conf = writeText "xl2tpd.conf" ''
    [lac upstream]
    lns = ${lns}
    require authentication = no
-    pppoptfile = ${writeText "ppp-options" ppp-options'}
+    pppoptfile = /run/${name}/ppp-options
    autodial = yes
    redial = yes
+    redial timeout = 1
+    max redials = 2 # this gives 1 actual retry, as xl2tpd can't count
  '';
-  control = "/run/xl2tpd/control-${name}";
-in
-longrun {
-  inherit name;
-  run = ''
-    mkdir -p /run/xl2tpd
+  control = "/run/${name}/control";
+in common {
+  inherit name debug username password lcpEcho ppp-options;
+  command = ''
    touch ${control}
-    exec ${xl2tpd}/bin/xl2tpd -D -p /run/xl2tpd/${name}.pid -c ${conf} -C ${control} 
+    exec ${xl2tpd}/bin/xl2tpd -D -p /run/${name}/${name}.pid -c ${conf} -C ${control} 
  '';
-  notification-fd = 10;
 }
@@ -1,51 +1,30 @@
 {
-  liminix
-, lib
-, ppp
-, pppoe
-, writeAshScript
-, serviceFns
+  lib,
+  liminix,
+  output-template,
+  ppp,
+  pppoe,
+  serviceFns,
+  svc,
+  writeAshScript,
+  callPackage
 } :
-{ interface, ppp-options }:
+{ interface,
+  ppp-options,
+  lcpEcho,
+  username,
+  password,
+  debug
+}:
 let
-  inherit (liminix.services) longrun;
  name = "${interface.name}.pppoe";
-  ip-up = writeAshScript "ip-up" {} ''
-    . ${serviceFns} 
-    (in_outputs ${name}
-     echo $1 > ifname
-     echo $2 > tty
-     echo $3 > speed
-     echo $4 > address
-     echo $5 > peer-address
-     echo $DNS1 > ns1
-     echo $DNS2 > ns2
-    )
-    echo >/proc/self/fd/10
+  common = callPackage ./common.nix { inherit svc; };
+
+  timeoutOpt = if lcpEcho.interval != null then "-T ${builtins.toString (4 * lcpEcho.interval)}" else "";
+in common {
+  inherit name debug username password lcpEcho ppp-options;
+  command = ''
+    ${ppp}/bin/pppd pty "${pppoe}/bin/pppoe ${timeoutOpt}  -I $(output ${interface} ifname)" file /run/${name}/ppp-options
  '';
-  ip6-up = writeAshScript "ip6-up" {} ''
-    . ${serviceFns} 
-    (in_outputs ${name}
-     echo $4 > ipv6-address
-     echo $5 > ipv6-peer-address
-    )
-    echo >/proc/self/fd/10
-  '';
-  ppp-options' = ppp-options ++ [
-    "ip-up-script" ip-up
-    "ipv6-up-script" ip6-up
-    "ipparam" name
-    "nodetach"
-    "usepeerdns"
-    "logfd" "2"
-  ];
-in
-longrun {
-  inherit name;
-  run = ''
-    . ${serviceFns} 
-    ${ppp}/bin/pppd pty "${pppoe}/bin/pppoe -I $(output ${interface} ifname)" ${lib.concatStringsSep " " ppp-options'}
-  '';
-  notification-fd = 10;
  dependencies = [ interface ];
 }
@@ -52,8 +52,6 @@ in {

    wan = {
      interface = mkOption { type = liminix.lib.types.interface; };
-      username = mkOption { type = types.str; };
-      password =  mkOption { type = types.str; };
      dhcp6.enable = mkOption { type = types.bool; };
    };

@@ -86,14 +84,7 @@ in {
      members = cfg.lan.interfaces;
    };

-    services.wan = svc.pppoe.build {
-      inherit (cfg.wan) interface;
-      ppp-options = [
-        "debug" "+ipv6" "noauth"
-        "name" cfg.wan.username
-        "password" cfg.wan.password
-      ];
-    };
+    services.wan = cfg.wan.interface;

    services.packet_forwarding = svc.network.forward.build { };

@@ -158,7 +149,6 @@ in {
      dependencies = [ config.services.wan ];
      name = "resolvconf";
      up = ''
-        . ${serviceFns}
        ( in_outputs ${name}
         echo "nameserver $(output ${config.services.wan} ns1)" > resolv.conf
         echo "nameserver $(output ${config.services.wan} ns2)" >> resolv.conf
@@ -81,7 +81,6 @@ in {
      # 'A list of DNS server' but doesn't say what separates the
      # list members. Assuming it's a space or other IFS character
      up = ''
-        . ${serviceFns}
        ( in_outputs ${name}
        for i in $(output ${config.services.dhcpc} dns); do
          echo "nameserver $i" > resolv.conf
@@ -0,0 +1,27 @@
+## Round Robin
+##
+## Given a list of services, run each in turn until it exits, then
+## runs the next.
+
+
+{ lib, pkgs, config, ...}:
+let
+  inherit (lib) mkOption types;
+  inherit (pkgs) liminix;
+  inherit (pkgs.liminix.services) longrun;
+in {
+  options = {
+    system.service.round-robin = mkOption {
+      description = "run services one at a time and failover to next";
+      type = liminix.lib.types.serviceDefn;
+    };
+  };
+  config.system.service.round-robin = config.system.callService ./service.nix {
+    services = mkOption {
+      type = types.listOf  liminix.lib.types.service;
+    };
+    name =  mkOption {
+      type = types.str;
+    };
+  };
+}
@@ -0,0 +1,32 @@
+{
+  liminix, lib, s6-rc-round-robin
+}:
+{ services, name} :
+let
+   inherit (liminix.services) oneshot longrun;
+   controlled-services = builtins.map
+    (s: s.overrideAttrs(o: { inherit controller; }))
+    services;
+  controller = let name' = "control-${name}"; in longrun {
+    name = name';
+    run = ''
+      in_outputs ${name'}
+      exec ${s6-rc-round-robin}/bin/s6-rc-round-robin \
+         -p ${proxy.name} \
+         ${lib.concatStringsSep " "
+           (builtins.map (f: f.name) controlled-services)}
+    '';
+  };
+  proxy = oneshot rec {
+    inherit name;
+    inherit controller;
+    buildInputs = controlled-services;
+    up = ''
+      echo start proxy ${name}
+      set -x
+      (in_outputs ${name}
+       cp -rv $(output_path ${controller} active)/* .
+      )
+    '';
+  };
+in proxy
@@ -1,4 +1,4 @@
-{ config, pkgs, lib, ... }:
+{ config, pkgs, lib, lim,  ... }:
 let
  inherit (pkgs)
    execline
@@ -8,8 +8,9 @@ let
    stdenvNoCC;
  inherit (lib.lists) unique concatMap;
  inherit (pkgs.pseudofile) dir symlink;
-  inherit (pkgs.liminix.services) oneshot bundle;
-
+  inherit (pkgs.liminix.services) oneshot bundle longrun;
+  inherit (lib) mkIf mkEnableOption mkOption types;
+  cfg = config.logging;
  s6-rc-db =
    let
      # In the default bundle we need to have all the services
@@ -106,11 +107,22 @@ let
      };
      notification-fd = { file = "3"; };
      run = {
-        file = ''
-              #!${execline}/bin/execlineb -P
-              ${execline}/bin/redirfd -w 1 /dev/null
-              ${execline}/bin/redirfd -rnb 0 fifo
-              ${s6}/bin/s6-log -bpd3 -- t /run/uncaught-logs
+        file =
+          let pstore = if cfg.pstore then "pipeline { tee /dev/pmsg0 }" else "";
+          in ''
+            #!${execline}/bin/execlineb -P
+            ${execline}/bin/redirfd -w 1 /dev/null
+            ${execline}/bin/redirfd -rnb 0 fifo
+            ${if cfg.shipping.enable then ''
+              pipeline { ${s6}/bin/s6-log -bpd3 -- ${cfg.script} 1 }
+              pipeline { ${pkgs.logshipper}/bin/logtap ${cfg.shipping.socket} logshipper-socket-event }
+              ${pstore}
+              ${s6}/bin/s6-log -- ${cfg.directory}
+            '' else ''
+              pipeline { ${s6}/bin/s6-log -bpd3 -- ${cfg.script} 1 }
+              ${pstore}
+              ${s6}/bin/s6-log --  ${cfg.directory}
+            ''}
          '';
        mode = "0755";
      };
@@ -203,7 +215,66 @@ let
      };
  };
 in {
+  options = {
+    logging = {
+      pstore = mkEnableOption "system logs in pstore for retention after reboot";
+      shipping = {
+        enable = mkEnableOption "unix socket for log shipping";
+        socket = mkOption {
+          description = "socket pathname"; type = types.path;
+          default = "/run/.log-shipping.sock";
+        };
+        service = mkOption {
+          description = "log shipper service";
+          type = pkgs.liminix.lib.types.service;
+        };
+      };
+      script = mkOption {
+        description = "\"log script\" used by fallback s6-log process";
+        type = types.str;
+        default = "p${config.hostname} t";
+      };
+      directory = mkOption {
+        description = "default log directory";
+        default = "/run/log";
+        type = types.path;
+      };
+    };
+  };
+  imports = [
+    ( {config, pkgs, lib, ...}:
+      let
+        cfg = config.logging;
+        pipeline = shipper: bundle {
+          name = "log-shipping-pipe";
+          contents = let
+            eat = longrun {
+              name = "log-shipping-pipe-eat";
+              run = ''
+                fdmove -c 12 1 \
+                ${pkgs.s6}/bin/s6-ipcserver ${cfg.shipping.socket} \
+                fdmove -c 1 12 \
+                cat
+              '';
+              producer-for = spew.name;
+            };
+            spew = shipper.override {
+              consumer-for ="log-shipping-pipe-eat";
+            };
+          in [ eat spew ];
+        };
+      in mkIf cfg.shipping.enable {
+        services.${cfg.shipping.service.name} = pipeline cfg.shipping.service;
+      }
+    )];
+
  config = {
+    kernel.config = mkIf cfg.pstore {
+      PSTORE = "y";
+      PSTORE_PMSG = "y";
+    };
+    programs.busybox.applets =  mkIf cfg.pstore [ "tee" ];
+    
    filesystem = dir {
      etc = dir {
        s6-rc = dir {
@@ -0,0 +1,86 @@
+## Secrets
+
+## various ways to manage secrets without writing them to the
+## nix store
+
+{ lib, pkgs, config, ...}:
+let
+  inherit (lib) mkOption types;
+  inherit (pkgs) liminix;
+  inherit (pkgs.liminix.services) longrun;
+in {
+  options.system.service.secrets = {
+    outboard = mkOption {
+      description = "fetch secrets from external vault with https";
+      type = liminix.lib.types.serviceDefn;
+    };
+    tang = mkOption {
+      description = "fetch secrets from encrypted local pathname, using tang";
+      type = liminix.lib.types.serviceDefn;
+    };
+    subscriber = mkOption {
+      description = "wrapper around a service that needs notifying (e.g. restarting) when secrets change";
+      type = liminix.lib.types.serviceDefn;
+    };
+
+  };
+  config.system.service.secrets = {
+    outboard = config.system.callService ./outboard.nix {
+      url = mkOption {
+        description = "source url";
+        type = types.strMatching "https?://.*";
+      };
+      username = mkOption {
+        description = "username for HTTP basic auth";
+        type = types.nullOr types.str;
+      };
+      password = mkOption {
+        description = "password for HTTP basic auth";
+        type = types.nullOr types.str;
+      };
+      name = mkOption {
+        description = "service name";
+        type = types.str;
+      };
+      interval =  mkOption {
+        type = types.int;
+        default = 30;
+        description = "how often to check the source, in minutes";
+      };
+    };
+    tang = config.system.callService ./tang.nix {
+      path = mkOption {
+        description = "encrypted source pathname";
+        type = types.path;
+      };
+      name = mkOption {
+        description = "service name";
+        type = types.str;
+      };
+      interval =  mkOption {
+        type = types.int;
+        default = 30;
+        description = "how often to check the source, in minutes";
+      };
+    };
+    subscriber = config.system.callService ./subscriber.nix {
+      watch = mkOption {
+        description = "secrets paths to subscribe to";
+        type = types.listOf (types.functionTo types.anything);
+      };
+      service = mkOption {
+        description = "subscribing service that will receive notification";
+        type = liminix.lib.types.service;
+      };
+      action = mkOption {
+        description = "how do we notify the service to regenerate its config";
+        default = "restart-all";
+        type = types.enum [
+          "restart" "restart-all"
+          "hup" "int" "quit" "kill" "term"
+          "winch" "usr1" "usr2"
+        ];
+      };
+    };
+  };
+}
@@ -0,0 +1,23 @@
+{
+  liminix, lib, json-to-fstree, serviceFns
+}:
+{ name, url, interval, username, password  } :
+let
+  inherit (liminix.services) oneshot longrun;
+  inherit (lib) optionalString;
+in longrun {
+  inherit name;
+  buildInputs = [ json-to-fstree ];
+  run = ''
+    ${optionalString (username != null) ''
+      export NETRC=$(mkstate ${name})/netrc
+      (echo default ; echo login ${username} ; echo password ${password} ) > $NETRC
+    ''}
+    ( in_outputs ${name}
+      while : ; do
+        ${json-to-fstree}/bin/json-to-fstree ${url} .
+        sleep ${builtins.toString (interval * 60)}
+      done
+    )
+  '';
+}
@@ -0,0 +1,51 @@
+{
+  liminix, lib, lim, s6, s6-rc, watch-outputs
+}:
+{ watch, service, action } :
+let
+  inherit (liminix.services) oneshot longrun;
+  inherit (builtins) length head toString;
+  inherit (lib) unique optional optionals;
+  inherit (service) name;
+
+  watched-services = unique (map (f: f "service") watch);
+  paths = unique (map (f: f "path") watch);
+
+  restart-flag = {
+    restart = "-r";
+    restart-all = "-R";
+    "hup" = "-s 1";
+    "int" = "-s 2";
+    "quit" = "-s 3";
+    "kill" = "-s 9";
+    "term" = "-s 15";
+    "winch" = "-s 28";
+    "usr1" = "-s 10";
+    "usr2" = "-s 12";
+  }.${action};
+
+  watched-service =
+    if length watched-services == 0
+    then null
+    else if length watched-services == 1
+    then head watched-services
+    else throw "cannot subscribe to more than one source service for secrets";
+
+  watcher = let name' = "restart-${name}"; in longrun {
+    name = name';
+    run = ''
+      dir=/run/service/${name}
+      echo waiting for $dir
+      if test -e $dir/notification-fd; then flag="-U"; else flag="-u"; fi
+      ${s6}/bin/s6-svwait $flag /run/service/${name} || exit
+      PATH=${s6-rc}/bin:${s6}/bin:$PATH
+      ${watch-outputs}/bin/watch-outputs ${restart-flag} ${name} ${watched-service.name} ${lib.concatStringsSep " " paths}
+    '';
+  };
+in service.overrideAttrs(o: {
+  buildInputs =  (lim.orEmpty o.buildInputs) ++
+                 optional (watched-service != null) watcher;
+  dependencies = (lim.orEmpty o.dependencies) ++
+                 optionals (watched-service != null)
+                   [ watcher watched-service ];
+})
@@ -0,0 +1,26 @@
+{
+  liminix, lib, json-to-fstree, serviceFns, tangc
+}:
+{ name, path, interval } :
+let
+  inherit (liminix.services) longrun;
+  inherit (lib) optionalString;
+in longrun {
+  inherit name;
+  buildInputs = [ json-to-fstree ];
+  notification-fd = 10;
+  run = ''
+    set -e
+    statedir=/run/${name}
+    mkdir -p -m 0700 $statedir
+    ( in_outputs ${name}
+      while : ; do
+        ${tangc}/bin/tangc decrypt < ${path} > $statedir/.input.json
+        mv $statedir/.input.json $statedir/input.json
+        ${json-to-fstree}/bin/json-to-fstree file://$statedir/input.json .
+        echo ready >&10
+        sleep ${builtins.toString (interval * 60)}
+      done
+    )
+  '';
+}
@@ -19,8 +19,9 @@ in {
      type = liminix.lib.types.serviceDefn;
    };
  };
+  config.programs.busybox.options.FEATURE_FANCY_ECHO = "y";
  config.system.service = {
-    ssh = liminix.callService ./ssh.nix {
+    ssh = config.system.callService ./ssh.nix {
      address = mkOption {
        type = types.nullOr types.str;
        default = null;
@@ -41,6 +42,16 @@ in {
        type = types.bool; default = false;
        description = "Allow remote hosts to connect to local forwarded ports (by default they are bound to loopback)";
      };
+      authorizedKeys = mkOption {
+        type = types.nullOr (liminix.lib.types.replacable (types.attrsOf (types.listOf types.nonEmptyStr)));
+        example = {
+          root = ["ssh-rsa AAAAB3N...aZaZ"];
+          alice = ["ssh-rsa AAAAB3N...qS4r"];
+          bob = [];
+        };
+        default = null;
+        description = "Authorized SSH public keys for each username. If this optin is provided it overrides any keys found in /home/{username}/.ssh";
+      };
      extraConfig = mkOption {
        type = types.separatedString " ";
        default = "";
@@ -2,11 +2,26 @@
  liminix
 , dropbear
 , lib
+, watch-ssh-keys
+}:
+{
+  address,
+  allowLocalPortForward,
+  allowPasswordLogin,
+  allowPasswordLoginForRoot,
+  allowRemoteConnectionToForwardedPorts,
+  allowRemotePortForward,
+  allowRoot,
+  authorizedKeys,
+  port,
+  extraConfig
 }:
-p :
 let
+  name = "sshd";
+  inherit (builtins) toString typeOf;
  inherit (liminix.services) longrun;
-  inherit (lib) concatStringsSep;
+  inherit (lib) concatStringsSep mapAttrs mapAttrsToList;
+  keydir = "/run/${name}/authorized_keys";
  options =
    [
      "-e" #  pass environment to child
@@ -15,26 +30,50 @@ let
      "-P /run/dropbear.pid"
      "-F" #  don't fork into background
    ] ++
-    (lib.optional (! p.allowRoot) "-w") ++
-    (lib.optional (! p.allowPasswordLogin) "-s") ++
-    (lib.optional (! p.allowPasswordLoginForRoot) "-g") ++
-    (lib.optional (! p.allowLocalPortForward) "-j") ++
-    (lib.optional (! p.allowRemotePortForward) "-k") ++
-    (lib.optional (! p.allowRemoteConnectionToForwardedPorts) "-a") ++
-    [(if p.address != null
-      then "-p ${p.address}:${p.port}"
-      else "-p ${builtins.toString p.port}")] ++
-    [p.extraConfig];
+    (lib.optional (! allowRoot) "-w") ++
+    (lib.optional (! allowPasswordLogin) "-s") ++
+    (lib.optional (! allowPasswordLoginForRoot) "-g") ++
+    (lib.optional (! allowLocalPortForward) "-j") ++
+    (lib.optional (! allowRemotePortForward) "-k") ++
+    (lib.optional (! allowRemoteConnectionToForwardedPorts) "-a") ++
+    (lib.optionals (authorizedKeys != null) ["-U" "${keydir}/%n"]) ++
+    [(if address != null
+      then "-p ${address}:${toString port}"
+      else "-p ${toString port}")] ++
+    [extraConfig];
+  isKeyservice = typeOf authorizedKeys == "lambda";
+  authKeysConcat =
+    if authorizedKeys != null && !isKeyservice
+    then mapAttrs
+      (n : v : concatStringsSep "\\n" v)
+      authorizedKeys
+    else {};
+  keyservice = longrun {
+    name = "${name}-watch-keys";
+    run = ''
+      mkdir -p ${keydir}
+      exec ${watch-ssh-keys}/bin/watch-ssh-keys -d ${keydir} ${authorizedKeys "service"} ${authorizedKeys "path"}
+    '';
+    dependencies = [ (authorizedKeys "service") ] ;
+  };
 in
 longrun {
-  name = "sshd";
+  inherit name;
  # we need /run/dropbear to point to hostkey storage, as that
  # pathname is hardcoded into the binary.
  # env -i clears the environment so we don't pass anything weird to
  # ssh sessions
  run = ''
    ln -s $(mkstate dropbear) /run
+    mkdir -p /run/${name}/authorized_keys
+    ${concatStringsSep "\n"
+      (mapAttrsToList
+        (n : v : "echo -e '${v}' > /run/${name}/authorized_keys/${n} ")
+        authKeysConcat
+      )
+     }
    . /etc/profile # sets PATH but do we need this?  it's the same file as ashrc
    exec env -i ENV=/etc/ashrc PATH=$PATH ${dropbear}/bin/dropbear ${concatStringsSep " " options}
  '';
+  dependencies = lib.optional isKeyservice keyservice;
 }
@@ -0,0 +1,21 @@
+{ liminix, certifix-client, svc, lib, writeText, serviceFns }:
+{
+  caCertificate,
+  secret,
+  subject,
+  serviceUrl
+}:
+let
+  inherit (builtins) filter isString split;
+  inherit (liminix.services) oneshot;
+  name = "certifix-${lib.strings.sanitizeDerivationName subject}";
+  caCertFile = writeText "ca.crt" caCertificate;
+  secretFile = writeText "secret" secret;
+in oneshot {
+  inherit name;
+  up = ''
+    (in_outputs ${name}
+     SSL_CA_CERT_FILE=${caCertFile} ${certifix-client}/bin/certifix-client --subject ${subject} --secret ${secretFile} --key-out key --certificate-out cert ${serviceUrl}
+    )
+  '';
+}
@@ -0,0 +1,40 @@
+
+{ lib, pkgs, config, ...}:
+let
+  inherit (lib) mkOption types;
+  inherit (pkgs) liminix;
+in
+{
+  options = {
+    system.service.tls-certificate = {
+      certifix-client =  mkOption {
+        type = liminix.lib.types.serviceDefn;
+      };
+    };
+  };
+  config.system.service.tls-certificate.certifix-client =
+    config.system.callService ./certifix-client.nix {
+      # this is probably read from files on the build machine,
+      # but are not named with ...File suffix because they are
+      # not files on the device (they get embedded into the store)
+      caCertificate = mkOption {
+        description = "CA certificate in PEM format. This must be the same CA as that which signed the certificate of the Certifix server";
+        type = types.str;
+      };
+      secret = mkOption {
+        description = "The shared secret to embed in signing request. This must match the secret configured in the Certifix service, otherwise it will refuse to sign the CSR.";
+        type = types.str;
+      };
+      subject = mkOption {
+        description = "Subject of the certificate request, as an X509 DN. The CN ('Common Name') you provide here is also used as the value of the SubjectAlternativeName extension.";
+        type = types.str;
+        example = "C=GB,ST=London,O=Liminix,OU=IT,CN=myhostname";
+      };
+      serviceUrl = mkOption {
+        description = "Certifix server endpoint URL";
+        type = types.str;
+        example = "https://certifix.lan:19613/sign";
+      };
+    };
+
+}
@@ -14,7 +14,7 @@ in {
    };
  };
  config = {
-    system.service.uevent-rule = liminix.callService ./rule.nix {
+    system.service.uevent-rule = config.system.callService ./rule.nix {
      serviceName = mkOption {
        description = "name of the service to run when the rule matches";
        type = types.str;
@@ -19,7 +19,7 @@ in
  options = {
    system.service.vlan = mkOption { type = liminix.lib.types.serviceDefn; };
  };
-  config.system.service.vlan = liminix.callService ./service.nix {
+  config.system.service.vlan = config.system.callService ./service.nix {
    ifname = mkOption {
      type = types.str;
      description = "interface name to create";
@@ -15,7 +15,7 @@ in
      type = liminix.lib.types.serviceDefn;
    };
  };
-  config.system.service.watchdog = liminix.callService ./watchdog.nix {
+  config.system.service.watchdog = config.system.callService ./watchdog.nix {
    watched = mkOption {
      description = "services to watch";
      type = types.listOf liminix.lib.types.service;
@@ -0,0 +1,49 @@
+{ config, pkgs, lib, ... }:
+let
+  inherit (pkgs) liminix;
+  inherit (lib) mkOption types;
+  huawei-cdc-ncm = pkgs.kmodloader.override {
+    targets = ["huawei_cdc_ncm"];
+    inherit (config.system.outputs) kernel;
+  };
+in {
+  imports = [
+    ../uevent-rule
+    ../mdevd.nix
+  ];
+
+  options = {
+    system.service.wwan.huawei-e3372 = mkOption {
+      type = liminix.lib.types.serviceDefn;
+    };
+  };
+  config = {
+    kernel.config = {
+      USB_NET_HUAWEI_CDC_NCM = "m";
+      USB_USBNET = "y";
+      USB_SERIAL = "y";
+      USB_SERIAL_OPTION = "y";
+    };
+    programs.busybox.applets = [
+      "insmod" "rmmod"
+    ];
+
+    # https://www.0xf8.org/2017/01/flashing-a-huawei-e3372h-4g-lte-stick-from-hilink-to-stick-mode/
+    system.service.wwan.huawei-e3372 =
+      let svc = config.system.callService ./huawei-e3372.nix {
+            apn = mkOption { type = types.str; };
+            username = mkOption { type = types.str; };
+            password = mkOption { type = types.str; };
+            authType = mkOption { type = types.enum [ "pap" "chap" ]; };
+          };
+      in
+        svc // {
+          build = args :
+            let args' = args // {
+                  dependencies = (args.dependencies or []) ++
+                                 [huawei-cdc-ncm];
+                };
+            in svc.build args' ;
+        };
+  };
+}
@@ -41,7 +41,10 @@ let
  };
  atz = oneshot rec {
    name = "modem-atz";
-    dependencies = [ modeswitch ];
+    # atz does not depend on modeswitch because modeswitch service
+    # is only running when the stick is in the wrong mode
+    dependencies = [ modeswitch.controller ];
+    buildInputs = [ modeswitch ];
    controller = (svc.uevent-rule.build {
      serviceName = name;
      terms = {
@@ -20,7 +20,7 @@ in
    type = liminix.lib.types.serviceDefn;
  };

-  config.boot.zyxel-dual-image = liminix.callService ./service.nix {
+  config.boot.zyxel-dual-image = config.system.callService ./service.nix {
    ensureActiveImage = mkOption {
      type = types.enum [ "primary" "secondary" ];
      default = "primary";
@@ -1,22 +1,27 @@
 final: prev:
 let
+  isCross = final.stdenv.buildPlatform != final.stdenv.hostPlatform;
+  crossOnly = pkg : amendFn : if isCross then (amendFn pkg) else pkg;
  extraPkgs = import ./pkgs/default.nix {
    inherit (final) lib callPackage;
  };
  inherit (final) fetchpatch;
-  lua_no_readline = prev.lua5_3.overrideAttrs(o: {
-    name = "lua-tty";
-    preBuild = ''
-      makeFlagsArray+=(PLAT="posix" SYSLIBS="-Wl,-E -ldl"  CFLAGS="-O2 -fPIC -DLUA_USE_POSIX -DLUA_USE_DLOPEN")
+  luaHost =
+    let
+      l = prev.lua5_3.overrideAttrs(o: {
+        name = "lua-tty";
+        preBuild = ''
+          makeFlagsArray+=(PLAT="posix" SYSLIBS="-Wl,-E -ldl"  CFLAGS="-O2 -fPIC -DLUA_USE_POSIX -DLUA_USE_DLOPEN")
    '';
-    # lua in nixpkgs has a postInstall stanza that assumes only
-    # one output, we need to override that if we're going to
-    # convert to multi-output
-    # outputs = ["bin" "man" "out"];
-    makeFlags =
-      builtins.filter (x: (builtins.match "(PLAT|MYLIBS).*" x) == null)
-        o.makeFlags;
-  });
+        # lua in nixpkgs has a postInstall stanza that assumes only
+        # one output, we need to override that if we're going to
+        # convert to multi-output
+        # outputs = ["bin" "man" "out"];
+        makeFlags =
+          builtins.filter (x: (builtins.match "(PLAT|MYLIBS).*" x) == null)
+            o.makeFlags;
+      });
+    in l.override { self = l; };

  s6 = prev.s6.overrideAttrs(o:
    let
@@ -41,20 +46,22 @@ let
        (if o ? patches then o.patches else []) ++
        (if patch_needed then [ patch ] else []);
    });
-  lua = let s = lua_no_readline.override { self = s; }; in s;
 in
 extraPkgs // {
  # liminix library functions
  lim = {
    parseInt = s: (builtins.fromTOML "r=${s}").r;
+    orEmpty = x: if x != null then x else [];
  };

  # keep these alphabetical

-  btrfs-progs = prev.btrfs-progs.override {
-    udevSupport = false;
-    udev = null;
-  };
+  btrfs-progs = crossOnly prev.btrfs-progs (
+    d: d.override {
+      udevSupport = false;
+      udev = null;
+    }
+  );

  chrony =
    let chrony' = prev.chrony.overrideAttrs(o: {
@@ -76,22 +83,33 @@ extraPkgs // {
      texinfo = null;
    };

+  # clevis without luks/tpm
+  clevis = crossOnly prev.clevis
+    (d: let c = d.overrideAttrs(o: {
+              outputs = ["out"];
+              preConfigure = ''
+                 rm -rf src/luks
+                 sed -i -e '/luks/d' src/meson.build
+              '';
+            }); in c.override {
+              asciidoc = null;
+              cryptsetup = null;
+              luksmeta = null;
+              tpm2-tools = null;
+            });
+
  # luarocks wants a cross-compiled cmake (which seems like a bug,
  # we're never going to run luarocks on the device, but ...)
  # but https://github.com/NixOS/nixpkgs/issues/284734
  # so we do surgery on the cmake derivation until that's fixed

-  cmake = prev.cmake.overrideAttrs(o:
-    # don't override the build cmake or we'll have to rebuild
-    # half the known universe to no useful benefit
-    if final.stdenv.buildPlatform != final.stdenv.hostPlatform
-    then {
+  cmake = crossOnly prev.cmake
+    (d: d.overrideAttrs(o: {
      preConfigure =
        builtins.replaceStrings
          ["$configureFlags"] ["$configureFlags $cmakeFlags"] o.preConfigure;
    }
-    else {}
-  );
+    ));

  dnsmasq =
    let d =  prev.dnsmasq.overrideAttrs(o: {
@@ -104,14 +122,39 @@ extraPkgs // {
      nettle = null;
    };

-  dropbear = prev.dropbear.overrideAttrs (o: {
-    postPatch = ''
-     (echo '#define DSS_PRIV_FILENAME "/run/dropbear/dropbear_dss_host_key"'
-      echo '#define RSA_PRIV_FILENAME "/run/dropbear/dropbear_rsa_host_key"'
-      echo '#define ECDSA_PRIV_FILENAME "/run/dropbear/dropbear_ecdsa_host_key"'
-      echo '#define ED25519_PRIV_FILENAME "/run/dropbear/dropbear_ed25519_host_key"') > localoptions.h
-    '';
-  });
+  dropbear = crossOnly prev.dropbear
+    (d: d.overrideAttrs (o: rec {
+      version = "2024.85";
+      src = final.fetchurl {
+        url = "https://matt.ucc.asn.au/dropbear/releases/dropbear-${version}.tar.bz2";
+        sha256 = "sha256-hrA2xDOmnYnOUeuuM11lxHc4zPkNE+XrD+qDLlVtpQI=";
+      };
+      patches =
+        # need to update nixpkgs patch for new version of dropbear
+        let passPath = final.runCommand "pass-path" {} ''
+        sed < ${builtins.head o.patches} -e 's,svr-chansession.c,src/svr-chansession.c,g' > $out
+      '';
+        in [
+          passPath
+          ./pkgs/dropbear/add-authkeyfile-option.patch
+        ];
+      postPatch = ''
+       (echo '#define DSS_PRIV_FILENAME "/run/dropbear/dropbear_dss_host_key"'
+        echo '#define RSA_PRIV_FILENAME "/run/dropbear/dropbear_rsa_host_key"'
+        echo '#define ECDSA_PRIV_FILENAME "/run/dropbear/dropbear_ecdsa_host_key"'
+        echo '#define ED25519_PRIV_FILENAME "/run/dropbear/dropbear_ed25519_host_key"') > localoptions.h
+      '';
+    }));
+
+  elfutils = crossOnly prev.elfutils
+    (d: let e = d.overrideAttrs(o: {
+              configureFlags = o.configureFlags ++[
+                "ac_cv_has_stdatomic=no"
+              ];
+            });
+        in e.override {
+          enableDebuginfod = false;
+        });

  hostapd =
    let
@@ -141,6 +184,10 @@ extraPkgs // {
      });
    in h.override { openssl = null; sqlite = null; };

+  # berlekey db needs libatomic which we haven't figured out yet.
+  # disabling it means we don't have arpd
+  iproute2 = crossOnly prev.iproute2 (d: d.override { db = null; });
+
  kexec-tools-static = prev.kexec-tools.overrideAttrs(o: {
    # For kexecboot we copy kexec into a ramdisk on the system being
    # upgraded from. This is more likely to work if kexec is
@@ -159,8 +206,16 @@ extraPkgs // {
    ];
  });

-  luaFull = prev.lua;
-  inherit lua;
+  lua = crossOnly prev.lua5_3 (_: luaHost);
+
+  luaossl' = luaHost.pkgs.luaossl.overrideAttrs(o: {
+    patches = [
+      (fetchpatch {
+        url = "https://patch-diff.githubusercontent.com/raw/wahern/luaossl/pull/218.patch";
+        hash = "sha256-2GOliY4/RUzOgx3rqee3X3szCdUVxYDut7d+XFcUTJw=";
+      })
+    ] ++ final.lib.optionals (o ? patches) o.patches;
+  });

  mtdutils = prev.mtdutils.overrideAttrs(o: {
    patches = (if o ? patches then o.patches else []) ++ [
@@ -177,30 +232,59 @@ extraPkgs // {
    ];
  });

-  openssl = prev.openssl.overrideAttrs (o: {
-    # we want to apply
-    # https://patch-diff.githubusercontent.com/raw/openssl/openssl/pull/20273.patch";
-    # which disables overriding the -march cflags to the wrong values,
-    # but openssl is used for bootstrapping so that's easier said than
-    # done. Do it the ugly way..
-    postPatch =
-      o.postPatch
-      + (
-        with final;
-        lib.optionalString (
-          stdenv.buildPlatform != stdenv.hostPlatform
-        ) "\nsed -i.bak 's/linux.*-mips/linux-mops/' Configure\n"
-      );
-  });
+  openssl = crossOnly prev.openssl
+    (d: d.overrideAttrs (o:
+      with final; {
+      # we want to apply
+      # https://patch-diff.githubusercontent.com/raw/openssl/openssl/pull/20273.patch";
+      # which disables overriding the -march cflags to the wrong values,
+      # but openssl is used for bootstrapping so that's easier said than
+      # done. Do it the ugly way..
+      postPatch =
+        o.postPatch
+        +  ''
+           sed -i.bak 's/linux.*-mips/linux-mops/' Configure
+        '';
+      # openssl with threads requires stdatomic which drags in libgcc
+      # as a dependency
+      configureFlags = ["no-threads"] ++ o.configureFlags;
+
+      # don't need or want this bash script
+      postInstall = o.postInstall + "rm $bin/bin/c_rehash\n";
+      }
+    ));

  pppBuild = prev.ppp;

-  qemuLim = let q = prev.qemu.overrideAttrs (o: {
-    patches = o.patches ++ [
-      ./pkgs/qemu/arm-image-friendly-load-addr.patch
-    ];
-  }); in q.override { nixosTestRunner = true; sdlSupport = false; };
-
+  qemuLim =
+    let
+      inherit (final) lib;
+      q = prev.qemu.overrideAttrs (o: {
+        patches = o.patches ++ [
+          ./pkgs/qemu/arm-image-friendly-load-addr.patch
+          (final.fetchpatch {
+            url = "https://lore.kernel.org/qemu-devel/20220322154658.1687620-1-raj.khem@gmail.com/raw";
+            hash = "sha256-jOsGka7xLkJznb9M90v5TsJraXXTAj84lcphcSxjYLU=";
+          })
+        ];
+      });
+      overrides = {
+        nixosTestRunner = true;
+        hostCpuTargets = map (f: "${f}-softmmu") [
+          "arm" "aarch64" "mips" "mipsel"
+        ];
+        sdlSupport = false;
+        numaSupport = false;
+        seccompSupport = false;
+        usbredirSupport = false;
+        libiscsiSupport = false;
+        tpmSupport = false;
+        uringSupport = false;
+        capstoneSupport = false;
+      } // lib.optionalAttrs (lib.versionOlder lib.version "24.10") {
+        texinfo = null;
+      };
+    in q.override overrides;
  rsyncSmall =
    let
      r = prev.rsync.overrideAttrs (o: {
@@ -272,16 +356,16 @@ extraPkgs // {
    '';
  };

-  libusb1 =
-    let u = prev.libusb1.overrideAttrs(o: {
-          # don't use gcc libatomic because it vastly increases the
-          # closure size
-          preConfigure = "sed -i.bak /__atomic_fetch_add_4/c\: configure.ac";
-        });
-    in u.override {
-      enableUdev = final.stdenv.buildPlatform == final.stdenv.hostPlatform;
-      withDocs = false;
-    };
+  libusb1 = crossOnly prev.libusb1 (
+    d: let u = d.overrideAttrs(o: {
+             # don't use gcc libatomic because it vastly increases the
+             # closure size
+             preConfigure = "sed -i.bak /__atomic_fetch_add_4/c\: configure.ac";
+           });
+       in u.override {
+         enableUdev = false;
+         withDocs = false;
+       });

  util-linux-small = prev.util-linux.override {
    ncursesSupport = false;
@@ -291,4 +375,8 @@ extraPkgs // {
    translateManpages = false;
    capabilitiesSupport = false;
  };
+
+  xl2tpd = prev.xl2tpd.overrideAttrs(o: {
+    patches = [ ./pkgs/xl2tpd-exit-on-close.patch ];
+  });
 }
@@ -1,14 +1,25 @@
+servicedir:=$(shell mktemp -d)

-default: fs.lua init.lua nl.lua svc.lua net/constants.lua
+default: fs.lua init.lua nl.lua svc.lua process.lua net/constants.lua

-test:
+CHECK=fs.fnl init.fnl svc.fnl process.fnl
+
+check:
 	ln -s . anoia
+	fennel ./run-tests.fnl $(CHECK)
 	fennel test.fnl
-	fennel test-svc.fnl
+	fennel test-svc.fnl $(servicedir)
+	test -f $(servicedir)/fish
+	test "`cat $(servicedir)/fish`" = "food"
+	test -d $(servicedir)/nested/path
+	test "`cat $(servicedir)/nested/path/name`" = "value"
+	test "`cat $(servicedir)/nested/path/complex/attribute`" = "val"
+	test "`cat $(servicedir)/nested/path/complex/other`" = "42"
+

 net/constants.lua: net/constants.c
 	$(CC) -imacros sys/socket.h -imacros linux/netlink.h -E -P - < net/constants.c  | sed 's/ *$$//g' | cat -s > net/constants.lua


 %.lua: %.fnl
-	fennel --compile $< > $@
+	fennel --add-macro-path './assert.fnl'  --compile $< > $@
@@ -18,4 +18,8 @@
               (.. "\nexpected " ve# "\ngot " va#)
               ))))

-{ : expect : expect= }
+(fn define-tests [& body]
+  (when _G.RUNNING_TESTS
+    `(do ,(unpack body))))
+
+{ : define-tests : expect : expect=  }
@@ -1,4 +1,5 @@
 {
+  bc, # for tests
  fennel,
  stdenv,
  linotify,
@@ -11,7 +12,7 @@ in stdenv.mkDerivation {
  inherit pname;
  version = "0.1";
  src = ./.;
-  nativeBuildInputs = [ fennel cpio ];
+  nativeBuildInputs = [ fennel cpio bc ];
  buildInputs = with lua.pkgs; [ linotify lualinux ];
  outputs = [ "out" "dev" ];

@@ -1,4 +1,5 @@
 (local ll (require :lualinux))
+(import-macros { :  define-tests : expect : expect= } :anoia.assert)

 (local S_IFMT   0xf000)
 (local S_IFSOCK 0xc000)
@@ -9,6 +10,12 @@
 (local S_IFCHR  0x2000)
 (local S_IFIFO  0x1000)

+(macro errno-check [x]
+  `(match ,x
+     val# val#
+     (nil errno#) (assert nil (.. "system call failed, errno=" errno#))
+     ))
+
 (fn ifmt-bits [mode] (and mode (band mode 0xf000)))

 (fn file-type [pathname]
@@ -33,14 +40,15 @@
  (or (directory? pathname)
      (let [parent (string.gsub pathname "/[^/]+/?$" "")]
        (or (directory? parent) (mktree parent))
-        (assert (ll.mkdir pathname)))))
+        (errno-check (ll.mkdir pathname)))))

 (fn dir [name]
-  (let [dp (assert (ll.opendir name) name)]
+  (let [dp (errno-check (ll.opendir name) name)]
    (fn []
-      (match (ll.readdir dp)
-        (name type) (values name type)
-        (nil err) (do (if err (print err)) (ll.closedir dp) nil)))))
+      (case (ll.readdir dp)
+        (name filetype) (values name filetype)
+        (nil err) (do (if (> err 0) (print "ERR" err)) (ll.closedir dp) nil)
+        ))))

 (fn rmtree [pathname]
  (case (file-type pathname)
@@ -58,6 +66,22 @@
    unknown
    (error (.. "can't remove " pathname " of mode \"" unknown "\""))))

+;; lualinux doesn't publish access(2), this is not exactly
+;; the same but will suffice until we can add it
+(fn executable? [f]
+  (let [statbuf {}
+        stat (ll.lstat f statbuf 1)]
+    (and stat (> (band (. stat 3) 73) 0)))) ; \0111
+
+(fn find-executable [exe search-path]
+  (accumulate [full-path nil
+               p (string.gmatch search-path "(.-):")]
+    (or full-path (let [f (.. p "/" exe)] (and (executable? f) f)))))
+
+(define-tests
+  (let [p (find-executable "yes" (os.getenv "PATH"))]
+    (expect (string.match p "coreutils.+bin/yes$"))))
+

 {
 : mktree
@@ -65,5 +89,6 @@
 : directory?
 : dir
 : file-type
+ : find-executable
 :symlink (fn [from to] (ll.symlink from to))
 }
@@ -1,6 +1,14 @@
-(fn assoc [tbl k v]
+;; importing assert.fnl macros here would be circular, so we can't use
+;; the full test functionality
+(macro define-tests [& body]
+  (when _G.RUNNING_TESTS
+    `(do ,(unpack body))))
+
+(fn assoc [tbl k v & more]
  (tset tbl k v)
-  tbl)
+  (case more
+    [k v] (assoc tbl k v)
+    _ tbl))

 (fn merge [table1 table2]
  (collect [k v (pairs table2) &into table1]
@@ -34,31 +42,67 @@
               c (str:gmatch ".")]
    (+ (* h 33) (string.byte c))))

+(fn table= [a b]
+  (if (= a b)
+      true
+      (and (= (type a) :table) (= (type b) :table)
+           (accumulate [equal true
+                        k v1 (pairs a)
+                        &until (not equal)]
+             ;; all keys in a have the same value in a and b
+             (and equal
+                  (let [v2 (. b k)] (and v2 (table= v1 v2)))))
+           (accumulate [present true
+                        k _ (pairs b)
+                        &until (not present)]
+             ;; there are no keys in b which are not also in a
+             (and present (. a k))))))
+
+(define-tests
+ (assert (table= {:a 1 :b 2} {:b 2 :a 1}))
+ (assert (not (table= {:a 1 :b 2 :k :l}  {:b 2 :a 1})))
+ (assert (not (table= {:a 1 :b 2}  {:b 2 :a 1 :k :l})))
+
+ (assert (table= {:a 1 :b {:l 17}} {:b {:l 17} :a 1}))
+ (assert (table= {:a [4 5 6 7] } {:a [4 5 6 7]}))
+ (assert (not (table= {:a [4 5 6 7] } {:a [4 5 6 7 8]})))
+ (assert (not (table= {:a [4 5 7 6] } {:a [4 5 6 7 ]})))
+
+ (assert (table= {} {}))
+ )
+
+(fn dig [tree path]
+  (match path
+    [el & more] (dig (. tree el) more)
+    [el] (. tree el)
+    [] tree))
+
+(fn %% [fmt ...] (string.format fmt ...))

 (local
 base64-indices
- (doto [
-        "B" "C" "D" "E" "F" "G" "H" "I" "J" "K" "L" "M" "N" "O" "P"
-        "Q" "R" "S" "T" "U" "V" "W" "X" "Y" "Z" "a" "b" "c" "d" "e" "f"
-        "g" "h" "i" "j" "k" "l" "m" "n" "o" "p" "q" "r" "s" "t" "u" "v"
-        "w" "x" "y" "z" "0" "1" "2" "3" "4" "5" "6" "7" "8" "9" "-" "_"
-        ]
-   (tset 0 "A")))
+ (let [base [
+         "B" "C" "D" "E" "F" "G" "H" "I" "J" "K" "L" "M" "N" "O" "P"
+         "Q" "R" "S" "T" "U" "V" "W" "X" "Y" "Z" "a" "b" "c" "d" "e" "f"
+         "g" "h" "i" "j" "k" "l" "m" "n" "o" "p" "q" "r" "s" "t" "u" "v"
+         "w" "x" "y" "z" "0" "1" "2" "3" "4" "5" "6" "7" "8" "9"
+         ]]
+   {
+    :url
+    (merge (dup base)
+           { 0 "A"
+             62 "-"
+             63 "_" })
+    :standard
+    (merge (dup base)
+           { 0 "A"
+             62 "+"
+             63 "/" })
+    }))

-;; local function base64(s)
-;;    local byte, rep = string.byte, string.rep
-;;    local pad = 2 - ((#s-1) % 3)
-;;    s = (s..rep('\0', pad)):gsub("...", function(cs)
-;;       local a, b, c = byte(cs, 1, 3)
-;;       return bs[a>>2] .. bs[(a&3)<<4|b>>4] .. bs[(b&15)<<2|c>>6] .. bs[c&63]
-;;    end)
-;;    return s:sub(1, #s-pad) .. rep('=', pad)
-;; end

-(fn base64url [s]
-  "URL-safe Base64-encoded form of s (no trailing padding)"
+(fn base64-encode [s bs]
  (let [pad (- 2 (% (- (# s) 1)  3))
-        bs base64-indices
        blank (string.rep "\0" pad)
        s (-> (.. s blank)
              (: :gsub
@@ -69,13 +113,110 @@
                         (. bs (bor (lshift (band a 3) 4)  (rshift b 4)))
                         (. bs (bor (lshift (band b 15) 2) (rshift c 6)))
                         (. bs (band c 63)))))))]
-    (s:sub 1 (- (# s) pad))))
+    (.. (s:sub 1 (- (# s) pad))
+        (string.rep "=" pad))))
+
+
+(fn base64-decode [input rindices]
+  ;; take groups of 4 characters, reverse-look them up in base64-indices,
+  ;; convert to 24 bit number,
+  ;; convert to three characters
+
+  ;; things that might happen at the end of the string:
+  ;;  1) if the input string length was n * 3, it will have been
+  ;;  encoded as n * 4 base64 digits, no special handling required
+  ;;  2) if the input string length was (n * 3) + 2, the last 16 bits
+  ;;  are encoded into 3 base64 digits which are followed by an '='
+  ;;  3) if the input string length was (n * 3) + 1, the last 8 bits
+  ;;  are encoded into 2 base64 digits which are followed by '=='
+  ;;  0) there is never an '===' padding, because an 8 bit byte
+  ;; can never be encoded into a single 6 bit  base64 digit
+
+  (let [input (.. input (string.rep "=" (% (- 4 (# input)) 4)))
+        pad-chars (# (or (string.match input "(=+)$") ""))
+        trim-index (case pad-chars
+                     0 -1
+                     1 -2
+                     2 -3
+                     3 (assert nil "3 pad bytes??"))]
+    (->
+     (icollect [s (string.gmatch input "(....)")]
+       (let [(a b c d) (string.byte s 1 4)
+             ri (fn [x] (assert (. rindices x) (.. "invalid " x)))
+             n (bor (ri d)
+                    (lshift (ri c) 6)
+                    (lshift (ri b) 12)
+                    (lshift (ri a) 18))]
+         (string.pack "bbb"
+                      (rshift (band 0xff0000 n) 16)
+                      (rshift (band 0x00ff00 n) 8)
+                      (band 0x0000ff n))))
+     (table.concat "")
+     (string.sub 1 trim-index)
+     )))
+
+(fn base64 [alphabet-des]
+  (let [alphabet (or (. base64-indices alphabet-des) alphabet-des (. base64-indices :standard) )
+        ralphabet (doto
+                      (collect [k v (pairs alphabet)]
+                        (values (string.byte v) k))
+                    (tset (string.byte "=") 0))]
+    {
+     :encode (fn [_ str] (base64-encode str alphabet))
+     :decode (fn [_ str] (base64-decode str ralphabet))
+     }))
+
+(fn base64url [str]
+  ;; for backward-compatibility, this does not pad the encoded string
+  (let [encoded (: (base64 :url) :encode str)]
+    (pick-values 1 (string.gsub encoded "=+$" ""))))
+
+
+(define-tests
+  (let [{: view} (require :fennel)
+        b64 (base64 :url)]
+
+    (let [a (b64:decode "YWxsIHlvdXIgYmFzZQ==")]
+      (assert (= a "all your base") (view a)))
+    (let [a (b64:decode "ZmVubmVsIHRoaW5n")]
+      (assert (= a "fennel thing")  a))
+    (let [a (b64:decode "TWFueSBoYW5kcyBtYWtlIGxpZ2h0IHdvcms=")]
+      (assert (= a "Many hands make light work") (view  a)))
+    (let [a (b64:encode "hello world")]
+      (assert (= a "aGVsbG8gd29ybGQ=") a))
+
+    (fn check [plain enc]
+      (let [a (b64:encode plain)] (assert (= a enc) (.. "encode " a)))
+      (let [a (b64:decode enc)] (assert (= a plain) (.. "decode " a))))
+
+    (check ""  "")
+    (check "f"  "Zg==")
+    (check "fo"  "Zm8=")
+    (check "foo"  "Zm9v")
+    (check "foob"  "Zm9vYg==")
+    (check "fooba"  "Zm9vYmE=")
+    (check "foobar"  "Zm9vYmFy")
+
+    (let [x (b64:decode "REtOdUtNS05BNEJWLXdfcUhtNU9YV2liOUxkX3RTdVJTQWVUR0dkWldBdVEyaURObDZ2b3pSbEJwMzlzOEltdkhWdmpzZmMiLCJ5IjoiQVlDY1QwOGZrNFZWZ2lZSVIxbkU4UlJGaGZOSGdBUEFzckRITmJtRGNfUGtWZmdDR0xTMTIweU5SNncwdjd5RUY4WDN1OGpvazhkU0pqN0hnWjZCZHAzcSJ9LCJraWQiOiJlalVDaXBCUE9BeDRWQ1dQdUtkVGlYNDNadW5XTDNjSWN6V1h1RVZyTVNFIn0")]
+      (assert (string.match x "}$") x))
+
+    ))
+
+
+
+    ;; doesn't work if the padding is missing
+    ;; (let [a (from-base64 "TWFueSBoYW5kcyBtYWtlIGxpZ2h0IHdvcms")]
+    ;;   (assert (= a "Many hands make light work") (view  a)))
+;    ))
+


 {
 : assoc
+ : base64
 : base64url
 : basename
+ : dig
 : dirname
 : dup
 : file-exists?
@@ -83,4 +224,6 @@
 : merge
 : split
 : system
+ : table=
+ : %%
 }
@@ -0,0 +1,84 @@
+(local ll (require :lualinux))
+(local { : find-executable } (require :anoia.fs))
+(import-macros { : define-tests : expect : expect= } :anoia.assert)
+
+(macro errno-check [x]
+  `(match ,x
+     val# val#
+     (nil errno#) (assert nil (.. "system call failed, errno=" errno#))
+     ))
+
+(fn popen2 [pname argv envp]
+  (case (ll.pipe2)
+    (cin-s cin-d)
+    (match (ll.pipe2)
+      (cout-s cout-d)
+      (let [(pid err) (ll.fork)]
+        (if (not pid) (error (.. "error: " err))
+            (= pid 0)
+            (do
+              (ll.close cin-d)
+              (ll.close cout-s)
+              (ll.dup2 cin-s 0)
+              (ll.dup2 cout-d 1)
+              (ll.dup2 cout-d 2)
+              (ll.execve pname argv envp)
+              (error "execve failed"))
+            (> pid 0)
+            (do
+              (ll.close cin-s)
+              (ll.close cout-d)))
+        (values pid cin-d cout-s))
+      (nil err) (error (.. "popen pipe out: " err)))
+    (nil err) (error (.. "popen pipe in: " err))))
+
+(fn spawn [pname argv envp callback]
+  (let [(pid in out) (popen2 pname argv envp)
+        pollfds [
+                 (bor (lshift in 32) (lshift 4 16))
+                 (bor (lshift out 32) (lshift 1 16))
+                 ]]
+    (while (or (> (. pollfds 1) 0) (> (. pollfds 2) 0))
+      (ll.poll pollfds)
+      (if
+       (> (band (. pollfds 2) 0x11) 0)  ; POLLIN | POLLHUP
+       (if (not (callback :out out)) (tset pollfds 2 (lshift -1 32)))
+
+       (> (band (. pollfds 1) 4) 0)     ; POLLOUT
+       (if (not (callback :in in)) (tset pollfds 1 (lshift -1 32)))
+       ))
+
+    (match (ll.waitpid pid)
+      (0 status) false
+      (pid status) (rshift (band status 0xff00) 8)
+      (nil errno) (error (.. "waitpid: " errno)))))
+
+
+(define-tests
+  (var buf "4 * 6\n")                   ;; spawn bc to multiply two numbers
+  (let [out []
+        p (spawn
+           (assert (find-executable "bc" (os.getenv "PATH")))
+           ["bc"]
+           (ll.environ)
+           (fn [stream fd]
+             (match stream
+               :out (let [b (ll.read fd)]
+                      (table.insert out b)
+                      (> (# b) 0))
+               :in (if (> (# buf) 0)
+                       (let [bytes (ll.write fd buf)]
+                         (set buf (string.sub buf (+ bytes 1) -1))
+                         true)
+                       (do
+                         (ll.close fd)
+                         false))
+               :err (assert nil (ll.read fd)))))]
+    (expect= (table.concat out) "24\n"))
+  )
+
+
+{
+ : popen2
+ : spawn
+ }
@@ -0,0 +1,10 @@
+(local fennel (require :fennel))
+(local specials (require :fennel.specials))
+
+(local compiler-env
+       (doto (. (specials.make-compiler-env) :_G)
+         (tset "RUNNING_TESTS" true)))
+
+(each [_ f (ipairs arg)]
+  (print :testing f)
+  (fennel.dofile f { :correlate true :compilerEnv compiler-env }))
@@ -1,6 +1,6 @@
 (local inotify (require :inotify))
-(local { : file-exists? } (require :anoia))
-(local { : file-type : dir &as fs } (require :anoia.fs))
+(local { : file-exists? : dirname } (require :anoia))
+(local { : file-type : dir : mktree &as fs } (require :anoia.fs))

 (fn read-line [name]
  (with-open [f (assert (io.open name :r) (.. "can't open file " name))]
@@ -18,6 +18,15 @@
                     inotify.IN_CLOSE_WRITE)
    handle))

+(fn write-value [pathname value]
+  (mktree (dirname pathname))
+  (match (type value)
+    "string"
+    (with-open [f (assert (io.open pathname :w) (.. "can't open " pathname))]
+      (f:write value))
+    "table" (each [k v (pairs value)]
+              (write-value (.. pathname "/" k) v))))
+
 (fn read-value [pathname]
  (case (file-type pathname)
    nil nil
@@ -46,8 +55,10 @@
     :wait #(watcher:read)
     :ready? (fn [self]
               (and (has-file? "state") (not (has-file? ".lock"))))
-     :output (fn [_ filename]
-               (read-value (.. directory "/" filename)))
+     :output (fn [_ filename new-value]
+               (if new-value
+                   (write-value (.. directory "/" filename) new-value)
+                   (read-value (.. directory "/" filename))))
     :close #(watcher:close)
     : events
     }))
@@ -15,3 +15,12 @@
                        :3 {:attribute "a33"}
                        :5 {:attribute "a55"}
                        :6 {:attribute "a66"}})))
+
+(let [dir (. arg 1)
+      ex2 (svc.open dir)]
+  (ex2:output "fish" "food")
+  (ex2:output "nested/path/name" "value")
+  (ex2:output "nested/path/complex" {
+                                     :attribute "val"
+                                     :other "42"
+                                     }))
@@ -0,0 +1,89 @@
+(local { : view } (require :fennel))
+(local { : assoc : split } (require :anoia))
+
+(local ctx (require :openssl.ssl.context))
+(local csr (require :openssl.x509.csr))
+(local altname (require :openssl.x509.altname))
+(local pkey (require :openssl.pkey))
+(local xn (require :openssl.x509.name))
+
+(local http (require :fetch))
+
+(macro ncall [f]
+  `(case ,f
+     ok# ok#
+     (nil err#) (error err#)))
+
+
+(fn x509-name [subj]
+  (let [n (xn.new)]
+    (each [_ c (ipairs (split "," subj))]
+      (let [(k v) (string.match c "(.-)=(.+)")]
+        (n:add k v)))
+    n))
+
+(fn x509-altname [subj]
+  (let [an (altname.new)]
+    (each [_ c (ipairs (split "," subj))]
+      (let [(k v) (string.match c "(.-)=(.+)")]
+        (if (= k "CN") (an:add "DNS" v))))
+    an))
+
+(fn parse-args [args]
+  (case args
+    ["--secret" secret & rest]
+    (assoc (parse-args rest)
+           :secret (with-open [f (ncall (io.open secret :r))] (f:read "l")))
+
+    ["--subject" subject & rest]
+    (assoc (parse-args rest) :subject subject)
+
+    ["--key-out" pathname & rest]
+    (assoc (parse-args rest) :key-out pathname)
+
+    ["--certificate-out" pathname & rest]
+    (assoc (parse-args rest) :certificate-out pathname)
+
+    [server] { : server }
+    _ {}))
+
+
+(local options (parse-args arg))
+
+(fn private-key []
+  (pkey.new { :type :rsa :bits 1024 }))
+
+(fn signing-request [pk]
+  (doto (csr.new)
+    (: :setVersion 3)
+    (: :setSubject (x509-name options.subject))
+    (: :setSubjectAlt (x509-altname options.subject))
+    (: :setPublicKey pk)
+    (: :addAttribute :challengePassword [options.secret])
+    (: :sign pk)))
+
+
+(fn http-post [url body]
+  (match
+      (http.request "POST" url
+                    "" 0
+                    "application/x-pem-file"
+                    body)
+    s s
+    (nil code msg) (error (.. "Error " code " POST " url ": " msg))))
+
+
+(fn run []
+  (let [pk (private-key)
+        csr (signing-request pk)
+    ;; key-out (or options.key-out-handle io.stdout)
+    ;; cert-out (or options.cert-out-handle io.stdout)
+        cert (http-post options.server (csr:toPEM))]
+    (with-open [f (ncall (io.open options.key-out :w))]
+      (f:write (pk:toPEM :private)))
+    (with-open [f (ncall (io.open options.certificate-out :w))]
+      (f:write cert))
+    (print "done")))
+
+
+{ : run }
@@ -0,0 +1,36 @@
+{
+  fetchurl,
+  writeFennel,
+  fennel,
+  fennelrepl,
+  runCommand,
+  lua,
+  anoia,
+  lualinux,
+  fetch-freebsd,
+  openssl,
+  luaossl',
+  stdenv
+}:
+let name = "certifix-client";
+in stdenv.mkDerivation {
+  inherit name;
+  src = ./.;
+
+  buildInputs = [fetch-freebsd openssl lua];
+
+  buildPhase = "";
+  installPhase = ''
+    mkdir -p $out/bin
+    cp -p ${writeFennel name {
+      packages = [
+        fetch-freebsd
+        fennel
+        anoia
+        lualinux
+        luaossl'
+      ] ;
+      mainFunction = "run";
+    } ./${name}.fnl } $out/bin/${name}
+  '';
+}
@@ -0,0 +1,9 @@
+(local { : view } (require :fennel))
+(local ctx (require openssl.ssl.ctx))
+
+(fn run []
+  (print "hey"))
+
+
+
+{ : run }
@@ -14,6 +14,9 @@ in {
      uimage = callPackage ./kernel/uimage.nix { };
      kernel = callPackage ./kernel { };
    };
+    outputRef = service : path :
+      let h = { inherit service path; };
+      in x : h.${x};
    callService = path : parameters :
      let pkg = callPackage path {};
          checkTypes = t : p : typeChecked (builtins.toString path) t p;
@@ -30,7 +33,7 @@ in {
      };
    lib = {
      types =
-        let inherit (lib) types isDerivation;
+        let inherit (lib) mkOption types isDerivation;
        in  rec {
          service = types.package // {
            name = "service";
@@ -43,6 +46,10 @@ in {
            description = "parametrisable s6-rc service definition";
            check = x: lib.isAttrs x && x ? parameters && x ? build;
          };
+          replacable = t : types.either
+            t
+            # function might return a service or a path
+            (types.functionTo types.anything);
        };
      inherit typeChecked;
    };
@@ -53,7 +60,9 @@ in {
  # please keep the rest of this list alphabetised :-)

  anoia = callPackage ./anoia { };
+  certifix-client = callPackage ./certifix-client { };
  devout = callPackage ./devout { };
+  fetch-freebsd = callPackage ./fetch-freebsd { };
  fennel = callPackage ./fennel { };
  fennelrepl = callPackage ./fennelrepl { };
  firewallgen = callPackage ./firewallgen { };
@@ -63,11 +72,14 @@ in {
  hi = callPackage ./hi { };
  ifwait = callPackage ./ifwait { };
  initramfs-peek = callPackage ./initramfs-peek { };
+  incz = callPackage ./incz { };
+  json-to-fstree = callPackage ./json-to-fstree { };
  kernel-backport = callPackage ./kernel-backport { };
  kmodloader = callPackage ./kmodloader { };
  levitate = callPackage ./levitate { };
  libubootenv = callPackage ./libubootenv { };
  linotify = callPackage ./linotify { };
+  logshipper = callPackage ./logshipper { };
  lualinux = callPackage ./lualinux { };

  # we need to build real lzma instead of using xz, because the lzma
@@ -87,14 +99,17 @@ in {
  odhcp-script = callPackage ./odhcp-script { };
  odhcp6c = callPackage ./odhcp6c { };
  openwrt = callPackage ./openwrt { };
+  output-template = callPackage ./output-template { };
  ppp = callPackage ./ppp { };
  pppoe = callPackage ./pppoe { };
  preinit = callPackage ./preinit { };
  pseudofile = callPackage ./pseudofile { };
  routeros = callPackage ./routeros { };
+  rxi-json = callPackage ./rxi-json { };
  run-liminix-vm = callPackage ./run-liminix-vm { };
  s6-init-bin = callPackage ./s6-init-bin { };
  s6-rc-database = callPackage ./s6-rc-database { };
+  s6-rc-round-robin = callPackage ./s6-rc-round-robin { };
  s6-rc-up-tree = callPackage ./s6-rc-up-tree { };

  # schnapps is written by Turris and provides a high-level interface
@@ -103,14 +118,17 @@ in {
  # vendor OS, or even to derisk Liminix updates on that device
  schnapps = callPackage ./schnapps { };

+  seedrng = callPackage ./seedrng {};
  serviceFns = callPackage ./service-fns { };
  swconfig = callPackage ./swconfig { };
  systemconfig = callPackage ./systemconfig { };
+  tangc = callPackage ./tangc { };
  tufted = callPackage ./tufted { };
  uevent-watch = callPackage ./uevent-watch { };
  usb-modeswitch = callPackage ./usb-modeswitch { };
+  watch-outputs = callPackage ./watch-outputs { };
+  watch-ssh-keys = callPackage ./watch-ssh-keys { };
  writeAshScript = callPackage ./write-ash-script { };
  writeAshScriptBin = callPackage ./write-ash-script/bin.nix { };
  writeFennel = callPackage ./write-fennel { };
-  writeFennelScript = callPackage ./write-fennel-script { };
 }
@@ -129,12 +129,12 @@


 (fn unix-socket [name]
-  (let [addr (string.pack "=Hz" AF_LOCAL name)]
-    (case (ll.socket AF_LOCAL SOCK_STREAM 0)
-      fd (case (ll.bind fd addr)
-           0 (doto fd (ll.listen 32))
-           (nil err) (values nil err))
-      (nil err) (values nil err))))
+  (let [addr (string.pack "=Hz" AF_LOCAL name)
+        fd (check-errno (ll.socket AF_LOCAL SOCK_STREAM 0))]
+    (os.remove name)
+    (check-errno (ll.bind fd addr))
+    (check-errno (ll.listen fd 32))
+    fd))

 (fn pollfds-for [fds]
  (icollect [_ v (ipairs fds)]
@@ -173,10 +173,10 @@
    (nil err) (do (print err) false)))

 (fn open-netlink [groups]
-  (match (ll.socket AF_NETLINK SOCK_RAW NETLINK_KOBJECT_UEVENT)
-    fd (doto fd (ll.bind (string.pack "I2I2I4I4" ; family pad pid groups
-                                      AF_NETLINK 0 0 groups)))
-    (nil errno) (values nil errno)))
+  (let [fd (check-errno (ll.socket AF_NETLINK SOCK_RAW NETLINK_KOBJECT_UEVENT))]
+    (check-errno (ll.bind fd (string.pack "I2I2I4I4" ; family pad pid groups
+                                          AF_NETLINK 0 0 groups)))
+    fd))

 (fn event-loop []
  (let [fds {}]
@@ -0,0 +1,386 @@
+commit bd51aae2e40814ac2ae5801fd9f83f6a4a886fb1
+Author: Daniel Barlow <dan@telent.net>
+Date:   Fri Aug 23 11:33:24 2024 +0100
+
+    add -U otion to set path to authorized_keys file
+    
+    based on https://github.com/mkj/dropbear/pull/35
+    by Salvador Fandino sfandino@yahoo.com
+    
+    - Allow authorized keys inside dirs with the sticky bit set
+    
+    - Add option -U for customizing authorized_keys path
+    
+    - Updated for dropbear 2024.85 (source files moved to src/)
+    
+    - allow %u, %d, %n "format specifiers" in pathname so that the user's
+      username/homedir/uid can be embedded into the path
+
+diff --git a/Makefile.in b/Makefile.in
+index 5ebfca2..686fbfb 100644
+--- a/Makefile.in
+++ b/Makefile.in
+@@ -51,7 +51,7 @@ COMMONOBJS = $(patsubst %,$(OBJ_DIR)/%,$(_COMMONOBJS))
+ _SVROBJS=svr-kex.o svr-auth.o sshpty.o \
+ 		svr-authpasswd.o svr-authpubkey.o svr-authpubkeyoptions.o svr-session.o svr-service.o \
+ 		svr-chansession.o svr-runopts.o svr-agentfwd.o svr-main.o svr-x11fwd.o\
+-		svr-tcpfwd.o svr-authpam.o
+		svr-tcpfwd.o svr-authpam.o pathexpand.o
+ SVROBJS = $(patsubst %,$(OBJ_DIR)/%,$(_SVROBJS))
+ 
+ _CLIOBJS=cli-main.o cli-auth.o cli-authpasswd.o cli-kex.o \
+diff --git a/manpages/dropbear.8 b/manpages/dropbear.8
+index bdb2ea0..c8d450d 100644
+--- a/manpages/dropbear.8
+++ b/manpages/dropbear.8
+@@ -29,6 +29,9 @@ or automatically with the '-R' option. See "Host Key Files" below.
+ .B \-R
+ Generate hostkeys automatically. See "Host Key Files" below.
+ .TP
+.B \-U \fIauthorized_keys
+Absolute pathname to file containing authorized user keys. May contain the sequences %d, %n, %u which are expanded to the user's home directory, username and numeric uid respectively. Default '%d/.ssh/authorized_keys'.
+.TP
+ .B \-F
+ Don't fork into background.
+ .TP
+diff --git a/src/pathexpand.c b/src/pathexpand.c
+new file mode 100644
+index 0000000..2028733
+--- /dev/null
+++ b/src/pathexpand.c
+@@ -0,0 +1,132 @@
+#include <limits.h>
+#include <string.h>
+#include <stdio.h>
+
+#ifdef TEST_PATHEXPAND
+
+/* to run tests:
+  gcc -Wall -o pathexpand -D TEST_PATHEXPAND=1 src/pathexpand.c && ./pathexpand
+*/
+
+char * pathexpand(char *relfilename);
+
+
+#define m_malloc(c) malloc(c)
+#define m_strdup(c) strdup(c)
+
+struct session {
+    struct AuthState {
+	char * pw_dir;
+	char * pw_name;
+	uid_t pw_uid;
+    } authstate;
+};
+
+struct session ses = {
+    .authstate = {
+	.pw_dir = "/home/dan",
+	.pw_name = "dan",
+	.pw_uid = 12345,
+    }
+};
+
+int exit_status = 0;
+
+int expect_expansion(char * input, char * expected) {
+    char *actual = pathexpand(input);
+    if(strcmp(actual, expected) != 0) {
+	printf("expected %s for %s, got %s\n", expected, input, actual);
+	exit_status++;
+    }
+    free(actual);
+    return exit_status;
+}
+
+int main(int argc, char *argv[]) {
+    for(int i = 1; i < argc; i++) {
+	char *actual =  pathexpand(argv[i]);
+	printf("%s => %s\n", argv[i], pathexpand(argv[i]));
+	free(actual);
+    }
+
+    /* a string without % is unaltered */
+    expect_expansion("hello", "hello");
+
+    /* discards single trailing % */
+    expect_expansion("hello%", "hello");
+
+    /* %% is transformed to % */
+    expect_expansion("hello%%", "hello%");
+    expect_expansion("hello%%goodbye", "hello%goodbye");
+
+    /* %u is transformed to uid */
+    expect_expansion("/run/user/%u/authorized_keys", "/run/user/12345/authorized_keys");
+    /* % sequences work when at start of string */
+    expect_expansion("%u/authorized_keys", "12345/authorized_keys");
+
+    /* %d expands to home directory */
+    expect_expansion("%d/.ssh", "/home/dan/.ssh");
+
+    /* %n expands to username */
+    expect_expansion("/tmp/%n/.ssh", "/tmp/dan/.ssh");
+
+    /* unrecognised specifiers are discarded */
+    expect_expansion("/hi/%q/.ssh", "/hi//.ssh");
+
+    exit(exit_status);
+}
+
+#else
+#include "session.h"
+#include "debug.h"
+#endif
+#define NUMLEN(c) strlen(#c)
+
+char * pathexpand(char *relfilename)
+{
+    char * filename;
+    int len;
+
+    len = strlen(relfilename);
+    for(char *p = relfilename; p; p = strchr(p, '%')) {
+	switch(*(p+1)) {
+	case 'd': len += strlen(ses.authstate.pw_dir); break;
+	case 'n': len += strlen(ses.authstate.pw_name); break;
+	case 'u': len += NUMLEN(INT_MAX); break;
+	}
+	if(*(p+1) == '\0') break;
+	p=p+2;
+    }
+    filename = m_malloc(len+1);
+    filename[0] = '\0';
+
+    char *start = relfilename;
+    char *out = filename;
+    char *p = relfilename;
+    do {
+	p = strchrnul(start, '%');
+	strncat(out, start, p - start);
+
+	if(*p == '\0') break;
+
+	switch(*(p+1)) {
+	case '\0':
+	    p++; break;
+	case 'd':
+	    strcat(out, ses.authstate.pw_dir); break;
+	case 'n':
+	    strcat(out, ses.authstate.pw_name); break;
+	case 'u':
+	    snprintf(out + strlen(out),
+		     NUMLEN(INT_MAX),
+		     "%d",
+		     ses.authstate.pw_uid);
+	    break;
+	case '%':
+	    strcat(out, "%"); break;
+	}
+	start = p+2;
+    }
+    while (*p);
+    return filename;		/* caller must free */
+}
+diff --git a/src/runopts.h b/src/runopts.h
+index 1c88b5c..707008f 100644
+--- a/src/runopts.h
+++ b/src/runopts.h
+@@ -128,7 +128,8 @@ typedef struct svr_runopts {
+ 	char * pidfile;
+ 
+ 	char * forced_command;
+-	char* interface;
+	char * authkeysfile;
+	char * interface;
+ 
+ #if DROPBEAR_PLUGIN
+ 	/* malloced */
+diff --git a/src/svr-authpubkey.c b/src/svr-authpubkey.c
+index 5d298cb..54502f4 100644
+--- a/src/svr-authpubkey.c
+++ b/src/svr-authpubkey.c
+@@ -73,7 +73,7 @@
+ 
+ static int checkpubkey(const char* keyalgo, unsigned int keyalgolen,
+ 		const unsigned char* keyblob, unsigned int keybloblen);
+-static int checkpubkeyperms(void);
+static int checkpubkeyperms(char *filename, char *base);
+ static void send_msg_userauth_pk_ok(const char* sigalgo, unsigned int sigalgolen,
+ 		const unsigned char* keyblob, unsigned int keybloblen);
+ static int checkfileperm(char * filename);
+@@ -431,6 +431,7 @@ out:
+ 	return ret;
+ }
+ 
+extern char *pathexpand(char *input);
+ 
+ /* Checks whether a specified publickey (and associated algorithm) is an
+  * acceptable key for authentication */
+@@ -458,19 +459,12 @@ static int checkpubkey(const char* keyalgo, unsigned int keyalgolen,
+ 		dropbear_exit("Failed to set euid");
+ 	}
+ #endif
+	filename = pathexpand(svr_opts.authkeysfile);
+
+ 	/* check file permissions, also whether file exists */
+-	if (checkpubkeyperms() == DROPBEAR_FAILURE) {
+-		TRACE(("bad authorized_keys permissions, or file doesn't exist"))
+	if (checkpubkeyperms(filename, ses.authstate.pw_dir) == DROPBEAR_FAILURE) {
+		TRACE(("bad authorized keys permissions on %s, or file doesn't exist", filename))
+ 	} else {
+-		/* we don't need to check pw and pw_dir for validity, since
+-		 * its been done in checkpubkeyperms. */
+-		len = strlen(ses.authstate.pw_dir);
+-		/* allocate max required pathname storage,
+-		 * = path + "/.ssh/authorized_keys" + '\0' = pathlen + 22 */
+-		filename = m_malloc(len + 22);
+-		snprintf(filename, len + 22, "%s/.ssh/authorized_keys",
+-					ses.authstate.pw_dir);
+-
+ 		authfile = fopen(filename, "r");
+ 		if (!authfile) {
+ 			TRACE(("checkpubkey: failed opening %s: %s", filename, strerror(errno)))
+@@ -486,7 +480,7 @@ static int checkpubkey(const char* keyalgo, unsigned int keyalgolen,
+ 	if (authfile == NULL) {
+ 		goto out;
+ 	}
+-	TRACE(("checkpubkey: opened authorized_keys OK"))
+	TRACE(("checkpubkey: opened %s OK", filename))
+ 
+ 	line = buf_new(MAX_AUTHKEYS_LINE);
+ 	line_num = 0;
+@@ -524,53 +518,47 @@ out:
+ 
+ /* Returns DROPBEAR_SUCCESS if file permissions for pubkeys are ok,
+  * DROPBEAR_FAILURE otherwise.
+- * Checks that the user's homedir, ~/.ssh, and
+- * ~/.ssh/authorized_keys are all owned by either root or the user, and are
+ * Checks filename and its parent directories recursively until the
+ * base directory (usually ~/) or one of its ancestors (up to /) is
+ * reached.
+ * The files and directories must be all owned by root or the user, and be
+  * g-w, o-w */
+-static int checkpubkeyperms() {
+-
+-	char* filename = NULL;
+static int checkpubkeyperms(char *filename, char *base) {
+	char* path = NULL;
+ 	int ret = DROPBEAR_FAILURE;
+ 	unsigned int len;
+ 
+-	TRACE(("enter checkpubkeyperms"))
+-
+-	if (ses.authstate.pw_dir == NULL) {
+-		goto out;
+-	}
+	TRACE(("enter checkpubkeyperms(%s, %s)", filename, base))
+ 
+-	if ((len = strlen(ses.authstate.pw_dir)) == 0) {
+	if ((base == NULL) || (base[0] != '/') ||
+	    (filename == NULL) || (filename[0] != '/')) {
+		/* both filename and base must be absolute paths */
+ 		goto out;
+ 	}
+ 
+-	/* allocate max required pathname storage,
+-	 * = path + "/.ssh/authorized_keys" + '\0' = pathlen + 22 */
+-	len += 22;
+-	filename = m_malloc(len);
+-	strlcpy(filename, ses.authstate.pw_dir, len);
+	len = strlen(filename);
+	path = m_strdup(filename);
+ 
+-	/* check ~ */
+-	if (checkfileperm(filename) != DROPBEAR_SUCCESS) {
+-		goto out;
+-	}
+-
+-	/* check ~/.ssh */
+-	strlcat(filename, "/.ssh", len);
+-	if (checkfileperm(filename) != DROPBEAR_SUCCESS) {
+-		goto out;
+-	}
+	while (checkfileperm(len ? path : "/") == DROPBEAR_SUCCESS) {
+		/* check if we are on base trail and if this is the
+		 * case, return success */
+		if ((strncmp(base, path, len) == 0) &&
+		    (!len || (base[len] == '\0') || (base[len] == '/'))) {
+			ret = DROPBEAR_SUCCESS;
+			break;
+		}
+ 
+-	/* now check ~/.ssh/authorized_keys */
+-	strlcat(filename, "/authorized_keys", len);
+-	if (checkfileperm(filename) != DROPBEAR_SUCCESS) {
+-		goto out;
+		/* look for parent directory */
+		while (--len) {
+			if (path[len] == '/') {
+				path[len] = '\0';
+				break;
+			}
+		}
+ 	}
+ 
+-	/* file looks ok, return success */
+-	ret = DROPBEAR_SUCCESS;
+-
+ out:
+-	m_free(filename);
+	m_free(path);
+ 
+ 	TRACE(("leave checkpubkeyperms"))
+ 	return ret;
+@@ -596,7 +584,9 @@ static int checkfileperm(char * filename) {
+ 		TRACE(("wrong ownership"))
+ 	}
+ 	/* check permissions - don't want group or others +w */
+-	if (filestat.st_mode & (S_IWGRP | S_IWOTH)) {
+	/* (unless sticky dir, which is allowed) */
+	if ((filestat.st_mode & (S_IWGRP | S_IWOTH)) &&
+	    !(S_ISDIR(filestat.st_mode) && (filestat.st_mode & S_ISVTX))) {
+ 		badperm = 1;
+ 		TRACE(("wrong perms"))
+ 	}
+diff --git a/src/svr-runopts.c b/src/svr-runopts.c
+index c4f83c1..faddfa2 100644
+--- a/src/svr-runopts.c
+++ b/src/svr-runopts.c
+@@ -147,6 +147,7 @@ void svr_getopts(int argc, char ** argv) {
+ 	char* maxauthtries_arg = NULL;
+ 	char* reexec_fd_arg = NULL;
+ 	char* keyfile = NULL;
+	char* authkeysfile = NULL;
+ 	char c;
+ #if DROPBEAR_PLUGIN
+         char* pubkey_plugin = NULL;
+@@ -173,6 +174,8 @@ void svr_getopts(int argc, char ** argv) {
+ 	svr_opts.hostkey = NULL;
+ 	svr_opts.delay_hostkey = 0;
+ 	svr_opts.pidfile = expand_homedir_path(DROPBEAR_PIDFILE);
+	svr_opts.authkeysfile = "%d/.ssh/authorized_keys";
+
+ #if DROPBEAR_SVR_LOCALANYFWD
+ 	svr_opts.nolocaltcp = 0;
+ #endif
+@@ -322,6 +325,9 @@ void svr_getopts(int argc, char ** argv) {
+ 				case 'u':
+ 					/* backwards compatibility with old urandom option */
+ 					break;
+				case 'U':
+					next = &authkeysfile;
+					break;
+ #if DROPBEAR_PLUGIN
+                                 case 'A':
+                                         next = &pubkey_plugin;
+@@ -372,6 +378,10 @@ void svr_getopts(int argc, char ** argv) {
+ 				addhostkey(keyfile);
+ 				keyfile = NULL;
+ 			}
+			if (authkeysfile) {
+				svr_opts.authkeysfile = m_strdup(authkeysfile);
+				authkeysfile = NULL;
+			}
+ 		}
+ 	}
+ 
@@ -27,17 +27,28 @@ in writeScriptBin "fennelrepl" ''
    package.path = ${lib.strings.escapeShellArg luapath} .. ";" .. package.path
    package.cpath = ${lib.strings.escapeShellArg luacpath} .. ";" .. (package.cpath or "")
    local fennel = require "fennel"
+    local specials = require("fennel.specials")
    table.insert(package.loaders or package.searchers,1, fennel.searcher)
    fennel['macro-path'] = "${anoia.dev}/share/lua/${lua.luaversion}/?.fnl;" .. fennel['macro-path']

+    local function eval_as_test(f)
+      local g = (specials["make-compiler-env"]())._G
+      g["RUNNING_TESTS"] = true
+      return fennel.dofile(f, {correlate = true, compilerEnv = g})
+    end
+
    local more_fennel = os.getenv("FENNEL_PATH")
    if more_fennel then
-        fennel.path = more_fennel .. ";" .. fennel.path
+      fennel.path = more_fennel .. ";" .. fennel.path
    end
    if #arg > 0 then
-       script = table.remove(arg, 1)
-       fennel.dofile(script, {correlate = true},  arg)
+      if arg[1] == '--test' then
+        eval_as_test(arg[2])
+      else
+        script = table.remove(arg, 1)
+        fennel.dofile(script, {correlate = true},  arg)
+      end
    else
-        fennel.repl()
+      fennel.repl()
    end
  ''
@@ -0,0 +1,31 @@
+{
+  cmake,
+  stdenv,
+  openssl,
+  lua,
+  fetchFromGitHub
+}: stdenv.mkDerivation {
+  pname = "fetch-freebsd";
+  version = "v12.0.11";
+  src = fetchFromGitHub {
+    owner= "jrmarino";
+    repo = "fetch-freebsd";
+    rev =  "v12.0.11";
+    hash = "sha256-nLNqjQFV9x2NntBdUlabxjS9q+er28zi8uXjWvCK2Ps=";
+  };
+  cmakeFlags = [
+    "-DFETCH_PROGRAM=OFF"
+    "-DFETCH_LIBRARY=ON"
+    "-DUSE_SYSTEM_SSL=ON"
+  ];
+  postBuild = ''
+    $CC -shared -o fetch-lua.so ${./lua-glue.c} -I$src -Llibrary -lssl -lfetch -llua
+  '';
+  nativeBuildInputs = [ cmake ];
+  buildInputs = [ lua openssl ];
+  postInstall = ''
+    rm -r $out/lib/lib*.a
+    mkdir -p $out/lib/lua/${lua.luaversion}
+    install fetch-lua.so  $out/lib/lua/${lua.luaversion}/fetch.so
+  '';
+}
@@ -0,0 +1,114 @@
+#include <stdlib.h>
+#include <string.h>
+
+#include <lua.h>
+#include <lualib.h>
+#include <lauxlib.h>
+#include "fetch.h"
+
+/*
+  FIXME: "By default libfetch allows TLSv1 and newer when negotiating
+  the connecting with the remote peer.  You can change this behavior
+  by setting the SSL_NO_TLS1, SSL_NO_TLS1_1 and SSL_NO_TLS1_2
+  environment variables to disable TLS 1.0, 1.1 and 1.2 respectively."
+*/
+
+int lp_error(lua_State *L, int code, char *message) {
+    lua_pushnil(L);
+    lua_pushinteger(L, code);
+    lua_pushlstring(L, message, strlen(message));
+    return 3;
+}
+
+int lp_fetched(lua_State *L, FXRETTYPE fp, struct url_stat *stat) {
+    size_t bytes_read = 0;
+    char * content = 0;
+    if(stat->size > 0) {
+	content = (char *) malloc(stat->size);
+	es_read(fp, content,  stat->size, &bytes_read);
+
+	if(bytes_read != stat->size) {
+	    lp_error(L, -1, "bytes read does not match content-length");
+	}
+    } else {
+	/* no content-length header */
+	int block_size = 128;
+	int buf_size = block_size;
+	size_t bytes_read_block;
+	do {
+	    content = (char *) realloc(content, buf_size);
+	    es_read(fp, content + bytes_read,  block_size, &bytes_read_block);
+	    bytes_read += bytes_read_block;
+	    buf_size += block_size;
+	} while(bytes_read_block > 0);
+    }
+
+    lua_pushlstring(L, (const char *) content, bytes_read);
+    lua_newtable(L);
+    lua_pushliteral(L, "last-modified");
+    lua_pushinteger(L, stat->mtime);
+    lua_settable(L, -3);
+    /* free(content) ? */
+    return 2;
+}
+
+int lp_fetch(lua_State *L) {
+    const char * url_string = luaL_checkstring(L, 1);
+    const char * flags = luaL_optstring(L, 2, "");
+    time_t if_modified_since = luaL_optinteger(L, 3, 0);
+
+    struct url *url = fetchParseURL(url_string);
+    if(url == NULL) {
+	return lp_error(L, -1, "url not parseable");
+    }
+    url->ims_time = if_modified_since;
+    struct url_stat stat;
+
+    FXRETTYPE fp = fetchXGet(url, &stat, flags);
+    fetchFreeURL(url);
+    if(fp == NULL) {
+	return lp_error(L, fetchLastErrCode, fetchLastErrString);
+    }
+    return lp_fetched(L, fp, &stat);
+}
+
+int lp_request(lua_State *L) {
+    const char * request_method = luaL_checkstring(L, 1);
+    const char * url_string = luaL_checkstring(L, 2);
+    const char * flags = luaL_optstring(L, 3, "");
+    time_t if_modified_since = luaL_optinteger(L, 4, 0);
+    const char * content_type = lua_tostring(L, 5);
+    const char * body = lua_tostring(L, 6);
+
+    struct url *url = fetchParseURL(url_string);
+    if(url == NULL) {
+	return lp_error(L, -1, "url not parseable");
+    }
+    url->ims_time = if_modified_since;
+    struct url_stat stat;
+
+    FXRETTYPE fp = http_request_body(url,
+				     request_method,
+				     &stat,
+				     NULL, /* no proxy */
+				     flags,
+				     content_type,
+				     body);
+    fetchFreeURL(url);
+    if(fp == NULL) {
+	return lp_error(L, fetchLastErrCode, fetchLastErrString);
+    }
+    return lp_fetched(L, fp, &stat);
+}
+
+static const struct luaL_Reg funcs [] = {
+    {"fetch", lp_fetch},
+    {"request", lp_request},
+    {NULL, NULL}  /* sentinel */
+};
+
+
+int luaopen_fetch (lua_State *L) {
+    luaL_newlib(L, funcs);
+    return 1;
+}
@@ -31,7 +31,7 @@ let

  indent = text : indentLines 0 (splitString "\n" text);

-  dochain = { name, type, rules,
+  dochain = { name, type, family, rules,
              policy ? null,
              priority ? "filter",
              hook ? null } : ''
@@ -1,15 +1,14 @@
 {
  netlink-lua,
-  writeFennelScript,
+  writeFennel,
  runCommand,
  anoia,
 }:
 runCommand "ifwait" { } ''
  mkdir -p $out/bin
  cp -p ${
-    writeFennelScript "ifwait" [
-      anoia
-      netlink-lua
-    ] ./ifwait.fnl
+    writeFennel "ifwait" {
+      packages = [ anoia netlink-lua ];
+    } ./ifwait.fnl
  } $out/bin/ifwait
 ''
@@ -0,0 +1,38 @@
+{
+  fetchurl,
+  writeFennel,
+  fennel,
+  fennelrepl,
+  runCommand,
+  lua,
+  anoia,
+  lualinux,
+  stdenv
+}:
+let name = "incz";
+in stdenv.mkDerivation {
+  inherit name;
+  src = ./.;
+
+  buildInputs = [lua];
+  nativeBuildInputs = [fennelrepl];
+
+  buildPhase = ''
+    fennelrepl --test ./incz.fnl
+    cp -p ${writeFennel name {
+      packages = [
+        anoia
+        lualinux
+        fennel
+      ];
+      macros = [
+        anoia.dev
+      ];
+      mainFunction = "run";
+    } ./incz.fnl } ${name}
+  '';
+
+  installPhase = ''
+    install -D ${name} $out/bin/${name}
+  '';
+}
@@ -0,0 +1,58 @@
+(local { : base64 } (require :anoia))
+(local ll (require :lualinux))
+
+(fn index [str indexname]
+  (string.format "{\"index\":{\"_index\":%q}}\n%s" indexname str))
+
+(local crlf "\r\n")
+
+(fn chunk [str]
+  (let [len (# str)]
+    (string.format "%x%s%s%s" len crlf str crlf)))
+
+(fn http-header [host path auth]
+  (string.format
+   "POST %s HTTP/1.1\r
+Host: %s\
+Authorization: basic %s
+Transfer-Encoding: chunked\r
+\r
+"
+   path host
+   (let [b64 (base64 :url)] (b64:encode auth))))
+
+(fn format-timestamp [timestamp]
+  ;; I can't make zincsearch understand any epoch-based timestamp
+  ;; formats, so we are formatting dates as iso-8601 and damn the leap
+  ;; seconds :-(
+  (let [secs (- (tonumber (string.sub timestamp 1 16) 16)
+                (lshift 1 62))
+        nano (tonumber (string.sub timestamp 16 24) 16)
+        ts (+ (* secs 1000) (math.floor (/ nano 1000000)))]
+    (.. (os.date "!%FT%T" secs) "." nano "Z")))
+
+(fn process-line [indexname hostname line]
+  (let [(timestamp service msg) (string.match line "@(%x+) (%g+) (.+)$")]
+    (->
+     (string.format
+      "{%q:%q,%q:%q,%q:%q,%q:%q}\n"
+      "@timestamp" (format-timestamp timestamp)
+      :service service
+      :message msg
+      :host hostname)
+     (index indexname)
+     chunk)))
+
+(fn run []
+  (let [myhostname (with-open [h (io.popen "hostname" :r)] (h:read "l"))
+        (filename loghost credentials indexname) (table.unpack arg)]
+
+    (with-open [infile (assert (io.open filename :r))]
+      (ll.write 1 (http-header loghost "/api/_bulk" credentials))
+      (while (case (infile:read "l")
+               line (ll.write 1 (process-line indexname myhostname line))))
+      (ll.write 1 (chunk "")))
+    (io.stderr:write (ll.read 0))))
+
+
+{ : run }
@@ -0,0 +1,37 @@
+{
+  fetchurl,
+  writeFennel,
+  fennel,
+  fennelrepl,
+  runCommand,
+  lua,
+  anoia,
+  linotify,
+  lualinux,
+  fetch-freebsd,
+  openssl,
+  rxi-json,
+  stdenv
+}:
+let name = "json-to-fstree";
+in stdenv.mkDerivation {
+  inherit name;
+  src = ./.;
+
+  buildInputs = [fetch-freebsd rxi-json openssl lua];
+
+  buildPhase = "";
+  installPhase = ''
+    mkdir -p $out/bin
+    cp -p ${writeFennel name {
+      packages = [
+        fetch-freebsd
+        rxi-json
+        anoia
+        lualinux
+        linotify
+      ] ;
+      mainFunction = "run";
+    } ./${name}.fnl } $out/bin/${name}
+  '';
+}
@@ -0,0 +1,30 @@
+(local json (require :json))
+(local http (require :fetch))
+(local svc (require :anoia.svc))
+(local { : utime } (require :lualinux))
+
+(fn download [url dest]
+  (let [state (.. dest "/state")
+        previously (ll.lstat state 12)]
+    (match (http.fetch url "i" previously)
+      (nil 10 _) ; not modified
+      (print (.. url " not modified, already up to date"))
+
+      (nil code str)
+      (assert nil (.. "error " code ": " str))
+
+      (body { : last-modified })
+      (let [service (svc.open dest)
+            lock (.. dest "/.lock")]
+        (with-open [fout (io.open lock :w)] (fout:write ""))
+        (service:output "." (json.decode body))
+        (with-open [fout (io.open state :w)] (fout:write "ok"))
+        (utime state last-modified)
+        (os.remove lock))
+
+      )))
+
+(fn run [] (download (. arg 1) (. arg 2)))
+
+
+{ : run }
@@ -62,6 +62,7 @@ let
          ../../modules/busybox.nix
          base
          ({ ... } : paramConfig)
+          ({ ... } : { config = { logging.script = "t"; }; })
          ../../modules/s6
        ];
      };
--- a/Show More
+++ b/Show More