From 34442b6069756c6dd2eeb8f576a921436df18ae8 Mon Sep 17 00:00:00 2001
From: Daniel Barlow <dan@telent.net>
Date: Tue, 12 Mar 2024 23:41:46 +0000
Subject: [PATCH 1/6] failing test for ifwait

---
 tests/ci.nix                   |  1 +
 tests/updown/configuration.nix | 55 ++++++++++++++++++++++++++++++++++
 tests/updown/script.expect     | 29 ++++++++++++++++++
 tests/updown/test.nix          | 19 ++++++++++++
 4 files changed, 104 insertions(+)
 create mode 100644 tests/updown/configuration.nix
 create mode 100644 tests/updown/script.expect
 create mode 100644 tests/updown/test.nix

diff --git a/tests/ci.nix b/tests/ci.nix
index f923e85..888a2d6 100644
--- a/tests/ci.nix
+++ b/tests/ci.nix
@@ -8,4 +8,5 @@
   min-copy-closure =  import ./min-copy-closure/test.nix;
   fennel = import ./fennel/test.nix;
   tftpboot = import ./tftpboot/test.nix;
+  updown = import ./updown/test.nix;
 }
diff --git a/tests/updown/configuration.nix b/tests/updown/configuration.nix
new file mode 100644
index 0000000..38e52db
--- /dev/null
+++ b/tests/updown/configuration.nix
@@ -0,0 +1,55 @@
+{ config, pkgs, lib, ... } :
+let
+  inherit (pkgs.liminix.services) bundle oneshot longrun;
+  inherit (pkgs) serviceFns;
+  # EDIT: you can pick your preferred RFC1918 address space
+  # for NATted connections, if you don't like this one.
+  ipv4LocalNet = "10.8.0";
+  svc = config.system.service;
+
+in rec {
+  imports = [
+    ../../modules/bridge
+    ../../modules/dhcp6c
+    ../../modules/dnsmasq
+    ../../modules/firewall
+    ../../modules/hostapd
+    ../../modules/network
+    ../../modules/ssh
+    ../../modules/vlan
+    ../../modules/wlan.nix
+  ];
+  rootfsType = "jffs2";
+  hostname = "updown";
+
+  services.int = svc.network.address.build {
+    interface = svc.bridge.primary.build { ifname = "int"; };
+    family = "inet"; address = "${ipv4LocalNet}.1"; prefixLength = 16;
+  };
+
+  services.bridge =  svc.bridge.members.build {
+    primary = services.int;
+    members = with config.hardware.networkInterfaces;
+      [ lan ];
+  };
+
+  services.sshd = svc.ssh.build { };
+
+  # users.root = {
+  #   # EDIT: choose a root password and then use
+  #   # "mkpasswd -m sha512crypt" to determine the hash.
+  #   # It should start wirh $6$.
+  #   passwd = "$6$6HG7WALLQQY1LQDE$428cnouMJ7wVmyK9.dF1uWs7t0z9ztgp3MHvN5bbeo0M4Kqg/u2ThjoSHIjCEJQlnVpDOaEKcOjXAlIClHWN21";
+  #   openssh.authorizedKeys.keys = [
+  #     # EDIT: you can add your ssh pubkey here
+  #     # "ssh-rsa AAAAB3NzaC1....H6hKd user@example.com";
+  #   ];
+  # };
+
+  defaultProfile.packages = with pkgs; [
+    min-collect-garbage
+#    strace
+    #    ethtool
+    tcpdump
+  ];
+}
diff --git a/tests/updown/script.expect b/tests/updown/script.expect
new file mode 100644
index 0000000..89e60c1
--- /dev/null
+++ b/tests/updown/script.expect
@@ -0,0 +1,29 @@
+set timeout 10
+
+spawn socat unix-connect:vm/monitor -
+set monitor_id $spawn_id
+
+expect "(qemu)"
+send "set_link virtio-net-pci.1 off\n"
+expect "(qemu)"
+send "set_link virtio-net-pci.0 off\n"
+expect "(qemu)"
+send "c\r\n"
+spawn socat unix-connect:vm/console -
+set console_id $spawn_id
+
+expect "BusyBox" 
+expect "#" { send "PS1=RE\\ADY_\\ \r" }
+expect "READY_" { send "sleep 3\r" }
+expect "READY_" { send "ip link\r" }
+expect "READY_" { send "cat /sys/class/net/lan/operstate\r" }
+expect {
+  "down" { }
+  "up" { exit 1 }
+}
+expect "READY_" { send "s6-rc -a -u change\r"  }
+expect {
+  "unable to take locks" { exit 1 }
+  "READY_" { send "hostname\r" }
+}
+expect "updown"
diff --git a/tests/updown/test.nix b/tests/updown/test.nix
new file mode 100644
index 0000000..589e383
--- /dev/null
+++ b/tests/updown/test.nix
@@ -0,0 +1,19 @@
+{
+  liminix
+, nixpkgs
+}:
+let img = (import liminix {
+      device = import "${liminix}/devices/qemu/";
+      liminix-config = ./configuration.nix;
+    }).outputs.vmroot;
+    pkgs = import <nixpkgs> { overlays = [(import ../../overlay.nix)]; };
+in pkgs.runCommand "check" {
+  nativeBuildInputs = with pkgs; [
+    expect
+    socat
+  ] ;
+} ''
+mkdir vm
+${img}/run.sh --flag -S --background ./vm
+expect ${./script.expect} | tee $out
+''

From af52aafc845194b0a6b3aa0e748105b4e53701e8 Mon Sep 17 00:00:00 2001
From: Daniel Barlow <dan@telent.net>
Date: Sat, 16 Mar 2024 20:22:30 +0000
Subject: [PATCH 2/6] deep thoughts

---
 THOUGHTS.txt | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/THOUGHTS.txt b/THOUGHTS.txt
index b4200f8..0bb66b1 100644
--- a/THOUGHTS.txt
+++ b/THOUGHTS.txt
@@ -4322,6 +4322,16 @@ set_link virtio-net-pci.0 on
 
 See if both devices are bridge members
 
+disable again,check if back to starting position
+
+
+Wed Mar 13 00:00:16 GMT 2024
+
+aside: "trigger" is the least bad word I've thought of so far for
+these services that stop/start other services
+
+telent: yeah, in general 'ps afuxww' (or s6-ps -H :)) is the way to solve this, look for hung s6-rc processes and in particular their s6-svlisten1 children, where the command line will show what service is still waiting for readiness
+
 Wed Mar 20 19:34:36 GMT 2024
 
 Because I forgot hoe to rebuild rotuer, I tihnk it is time to improve

From fad0a47b7547252330e4f0931c651ef63f3615dd Mon Sep 17 00:00:00 2001
From: Daniel Barlow <dan@telent.net>
Date: Sat, 16 Mar 2024 20:23:18 +0000
Subject: [PATCH 3/6] add config.system.callService

this is like pkgs.callService except that it passes
config.system.service as a param so that the service
being defined can invoke other services

if this proves to be a good idea, all uses of
pkgs.callService should be changed to use it instead
---
 modules/base.nix | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/modules/base.nix b/modules/base.nix
index 8f53df2..4d3d8e4 100644
--- a/modules/base.nix
+++ b/modules/base.nix
@@ -29,6 +29,10 @@ in {
     services = mkOption {
       type = types.attrsOf type_service;
     };
+    system.callService = mkOption {
+      type = types.functionTo (types.functionTo types.anything);
+    };
+
     filesystem = mkOption {
       type = types.anything;
       description = ''
@@ -111,6 +115,31 @@ in {
       "fw_devlink=off"
     ] ++ lib.optional (config.rootOptions != null) "rootflags=${config.rootOptions}";
 
+    system.callService = path : parameters :
+      let
+        typeChecked = caller: type: value:
+          let
+            inherit (lib) types mergeDefinitions;
+            defs = [{ file = caller; inherit value; }];
+            type' = types.submodule { options = type; };
+          in (mergeDefinitions [] type' defs).mergedValue;
+        cp = lib.callPackageWith(pkgs // { svc = config.system.service; });
+        pkg = cp path {};
+        checkTypes = t : p : typeChecked (builtins.toString path) t p;
+      in {
+        inherit parameters;
+        build = { dependencies ? [], ... } @ args :
+          let
+            s = pkg (checkTypes parameters
+              (builtins.removeAttrs args ["dependencies"]));
+          in s.overrideAttrs (o: {
+            dependencies = (builtins.map (d: d.name) dependencies) ++ o.dependencies;
+            buildInputs = dependencies ++ o.buildInputs;
+          });
+      };
+
+
+
     users.root = {
       uid = 0; gid= 0; gecos = "Root of all evaluation";
       dir = "/home/root/";

From 28a5dec7dd31834394748c9b19411715eeddc43c Mon Sep 17 00:00:00 2001
From: Daniel Barlow <dan@telent.net>
Date: Sat, 16 Mar 2024 20:41:13 +0000
Subject: [PATCH 4/6] implement ifwait trigger service and use in bridge

should we convert all ifwait uses to this trigger too? seems
reasonable
---
 modules/bridge/default.nix |  6 +++--
 modules/bridge/members.nix | 21 ++++++++++++------
 modules/ifwait/default.nix | 18 +++++++++++++++
 modules/ifwait/ifwait.nix  | 15 +++++++++++++
 tests/updown/script.expect | 45 +++++++++++++++++++++++++++++++++-----
 5 files changed, 91 insertions(+), 14 deletions(-)
 create mode 100644 modules/ifwait/default.nix
 create mode 100644 modules/ifwait/ifwait.nix

diff --git a/modules/bridge/default.nix b/modules/bridge/default.nix
index 48fdb12..32df4a5 100644
--- a/modules/bridge/default.nix
+++ b/modules/bridge/default.nix
@@ -14,6 +14,8 @@ let
   inherit (pkgs) liminix;
 in
 {
+  imports = [ ../ifwait ];
+
   options = {
     system.service.bridge = {
       primary = mkOption { type = liminix.lib.types.serviceDefn; };
@@ -27,7 +29,7 @@ in
         description = "bridge interface name to create";
       };
     };
-    members = liminix.callService ./members.nix {
+    members = config.system.callService ./members.nix {
       primary = mkOption {
         type = liminix.lib.types.interface;
         description = "primary bridge interface";
@@ -47,5 +49,5 @@ in
     # a better way to test for the existence of vlan config:
     # maybe the module should set an `enabled` attribute?
     BRIDGE_VLAN_FILTERING = "y";
-  };    
+  };
 }
diff --git a/modules/bridge/members.nix b/modules/bridge/members.nix
index ca1989e..a278730 100644
--- a/modules/bridge/members.nix
+++ b/modules/bridge/members.nix
@@ -2,6 +2,7 @@
   liminix
 , ifwait
 , lib
+, svc
 }:
 { members, primary } :
 
@@ -10,14 +11,20 @@ let
   inherit (liminix.services) bundle oneshot;
   inherit (lib) mkOption types;
   addif = member :
-    oneshot {
-      name = "${primary.name}.member.${member.name}";
-      up = ''
-        dev=$(output ${member} ifname)
-        ${ifwait}/bin/ifwait $dev running && ip link set dev $dev master $(output ${primary} ifname)
-      '';
-      down = "ip link set dev $(output ${member} ifname) nomaster";
+    # how do we get sight of services from here? maybe we need to
+    # implement ifwait as a regualr derivation instead of a
+    # servicedefinition
+    svc.ifwait.build {
+      state = "running";
+      interface = member;
       dependencies = [ primary member ];
+      service = oneshot {
+        name = "${primary.name}.member.${member.name}";
+        up = ''
+          ip link set dev $(output ${member} ifname) master $(output ${primary} ifname)
+        '';
+        down = "ip link set dev $(output ${member} ifname) nomaster";
+      };
     };
 in bundle {
   name = "${primary.name}.members";
diff --git a/modules/ifwait/default.nix b/modules/ifwait/default.nix
new file mode 100644
index 0000000..501f317
--- /dev/null
+++ b/modules/ifwait/default.nix
@@ -0,0 +1,18 @@
+{ config, pkgs, lib, ... } :
+let
+  inherit (pkgs) liminix;
+  inherit (lib) mkOption types;
+in {
+  options.system.service.ifwait =
+    mkOption { type = liminix.lib.types.serviceDefn; };
+
+  config.system.service.ifwait = config.system.callService ./ifwait.nix {
+    state = mkOption { type = types.str; };
+    interface = mkOption {
+      type = liminix.lib.types.interface;
+    };
+    service = mkOption {
+      type = liminix.lib.types.service;
+    };
+  };
+}
diff --git a/modules/ifwait/ifwait.nix b/modules/ifwait/ifwait.nix
new file mode 100644
index 0000000..4c28b52
--- /dev/null
+++ b/modules/ifwait/ifwait.nix
@@ -0,0 +1,15 @@
+{ ifwait, liminix } :
+{
+  state
+, interface
+, service
+}:
+let
+  inherit (liminix.services) longrun;
+in longrun {
+  name = "ifwait.${interface.name}";
+  buildInputs = [ service ];
+  run = ''
+    ${ifwait}/bin/ifwait -s ${service.name}  $(output ${interface} ifname) ${state}
+  '';
+}
diff --git a/tests/updown/script.expect b/tests/updown/script.expect
index 89e60c1..232f49a 100644
--- a/tests/updown/script.expect
+++ b/tests/updown/script.expect
@@ -13,9 +13,14 @@ spawn socat unix-connect:vm/console -
 set console_id $spawn_id
 
 expect "BusyBox" 
-expect "#" { send "PS1=RE\\ADY_\\ \r" }
-expect "READY_" { send "sleep 3\r" }
-expect "READY_" { send "ip link\r" }
+expect "#" { send "PS1=RE\\ADY_\\ ; stty -echo \r" }
+expect "READY_" { send "s6-rc -b -a list\r"  } ; # -b waits for s6-rc lock
+expect "READY_" { send "ls /sys/class/net/lan/master\r" }
+expect {
+  "No such file or directory" { }
+  timeout { exit 1 }
+}
+
 expect "READY_" { send "cat /sys/class/net/lan/operstate\r" }
 expect {
   "down" { }
@@ -24,6 +29,36 @@ expect {
 expect "READY_" { send "s6-rc -a -u change\r"  }
 expect {
   "unable to take locks" { exit 1 }
-  "READY_" { send "hostname\r" }
+  "READY_" { send "\r" }
 }
-expect "updown"
+
+set spawn_id $monitor_id
+send "\r"
+expect "(qemu)"
+send "set_link virtio-net-pci.1 on\n"
+expect "(qemu)"
+send "set_link virtio-net-pci.0 on\n"
+expect "(qemu)"
+set spawn_id $console_id
+
+expect "entered forwarding state"
+send "\r"
+expect "READY_" { send "cat /sys/class/net/lan/operstate\r" }
+expect {
+  "down" {  exit 1 }
+  "up" { }
+}
+
+expect "READY_" { send "cat /sys/class/net/lan/master/uevent\r" }
+expect {
+  "INTERFACE=int" { }
+  timeout { exit 1 }
+}
+
+expect "READY_" { send "s6-rc listall int.link.a.10.8.0.1.member.lan.link ; hostname\r"  }
+
+expect {
+  "updown"  {}
+  timeout { exit 1 }
+}
+

From 77f1a783310075bbc205b7f4ff875933687758ad Mon Sep 17 00:00:00 2001
From: Daniel Barlow <dan@telent.net>
Date: Sat, 16 Mar 2024 23:09:41 +0000
Subject: [PATCH 5/6] ifwait block if s6-rc lock is held

otherwise it doesn't trigger the service if something else is
slow to start
---
 pkgs/ifwait/ifwait.fnl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkgs/ifwait/ifwait.fnl b/pkgs/ifwait/ifwait.fnl
index 9af41f9..cc5e76c 100644
--- a/pkgs/ifwait/ifwait.fnl
+++ b/pkgs/ifwait/ifwait.fnl
@@ -37,8 +37,8 @@
   (when (not (= up wanted?))
     (set up
          (if wanted?
-             (pcall system (.. "s6-rc -u change " service))
-             (not (pcall system (.. "s6-rc -d change " service)))))
+             (pcall system (.. "s6-rc -b -u change " service))
+             (not (pcall system (.. "s6-rc -b -d change " service)))))
     ))
 
 (fn run [args event-fn]

From efcfdcc21d7d410673f04744d880e5ac5f17c759 Mon Sep 17 00:00:00 2001
From: Daniel Barlow <dan@telent.net>
Date: Thu, 28 Mar 2024 20:28:47 +0000
Subject: [PATCH 6/6] think

---
 THOUGHTS.txt | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)

diff --git a/THOUGHTS.txt b/THOUGHTS.txt
index 0bb66b1..6606706 100644
--- a/THOUGHTS.txt
+++ b/THOUGHTS.txt
@@ -4354,3 +4354,61 @@ just now but it does mean we can punt on specifying the device inside the
 liminix-config which is unreasonably circular.
 
 Maybe we'll just chuck a makefile in telent-nixos-config
+
+Fri Mar 22 22:14:32 GMT 2024
+
+For the service failover milestone we said
+
+a. A configuration demonstrating a service which is restarted when it crashes
+b. A failover config where service B runs iff service A is unavailable
+c. A config showing different pppd behaviour when interface is flakey (retry) vs ppp password is wrong (report error, wait for resolution)
+
+Sun Mar 24 23:41:27 GMT 2024
+
+TODO
+
+1) make liminix-rebuild bounce only affected services instead of
+  full reboot (what does it do about triggered services?)
+2) sniproxy
+
+3) see if arhcive still works. usb disk hotplug would be a good candidate for
+switching to triggers
+
+Mon Mar 25 19:35:47 GMT 2024
+
+to make the liminix-rebuild thing restart only affected services, it needs to
+know when the new service is not like the old one. By default it does not
+restart a service with a changed up/down/run script unless the name has
+also changed, so we need to figure out how to generate a "conversion"
+file with the services that are different
+
+pkgs/s6-rc-database/default.nix creates $out/compiled, we could add
+$out/hashes to this
+
+the other thing making this fun is that we will need to run `activate`
+(which is usually done in preinit) otherwise the new configuration's
+fhs directories won't exist.
+
+so the plan woyuld be
+
+in liminix-rebuild, when reboot was not chosen,
+
+- run activate
+- compare  /run/s6-rc/compiled/hashes (old services) with
+  /etc/s6-rc/compiled/hashes (new services)
+
+- whenever both files have the same column 1 and different
+column 2, add that name to restart list
+
+(need to turn restarts.fnl into a lua script)
+
+s6-rc-update /etc/s6-rc/compiled/hashes restarts
+
+Tue Mar 26 23:18:53 GMT 2024
+
+activate overwrites /etc/s6-rc/compiled, which is a problem because
+s6-rc-update expects to find the old compiled database here so that
+it can know what to update
+
+Maybe config.filesystem should specify /etc/s6-rc/compiled.new
+and something in early boot could symlink /etc/s6-rc/compiled to it