think

there was a race where ip-up could write ifname and then
ip6-up could write its outputs and then test ifname and
signal ready before ip-up had written the rest of its outputs

THOUGHTS.txt

@@ -7378,3 +7378,72 @@ for each service
     set tree = new tree
 wait for changes
 next iteration
+
+Fri Mar 28 23:25:49 GMT 2025
+
+the ppp connection doesn't come back after a&a reset the lns, this has
+happened twice now. Possibly-relevant log entries:
+
+
+2025-03-27 01:10:20.189135502 rotuer dhcp6c.prefix.dhcp6c.wan.link.pppoe.int.link.a.192.168.8.1 Executed "ip address change 2001:8b0:de3a:40de::1/64 dev int valid_lft 5682 preferred_lft 5682", exit code true
+2025-03-27 01:10:20.189182063 rotuer dhcp6c.addr.dhcp6c.wan.link.pppoe.wan.link.pppoe Executed "ip address change 2001:8b0:1111:1111:0:ffff:d9a9:14de/128 dev ppp0 valid_lft 5682 preferred_lft 2082", exit code true
+
+# am assuming these are just normal updates from remote dhcp6
+
+2025-03-27 01:11:19.179528428 rotuer wan.link.pppoe rcvd [LCP TermReq id=0x0]
+2025-03-27 01:11:19.179542189 rotuer wan.link.pppoe LCP terminated by peer
+2025-03-27 01:11:19.179545869 rotuer wan.link.pppoe Connect time 54273.0 minutes.
+2025-03-27 01:11:19.179550109 rotuer wan.link.pppoe Sent 3022278191 bytes, received 2202783935 bytes.
+2025-03-27 01:11:19.183056872 rotuer wan.link.pppoe sent [LCP TermAck id=0x0]
+# ppp died
+2025-03-27 01:11:19.183071352 rotuer dhcp6c.wan.link.pppoe odhcp6c[615]: carrier => 0 event on
+2025-03-27 01:11:19.183076072 rotuer dhcp6c.wan.link.pppoe odhcp6c[615]: Failed to send RS (Network unreachable)
+2025-03-27 01:11:19.183081592 rotuer dhcp6c.wan.link.pppoe odhcp6c[615]: Starting RELEASE transaction (timeout 4294967295s, max rc 5)
+2025-03-27 01:11:19.183086312 rotuer dhcp6c.wan.link.pppoe odhcp6c[615]: Send RELEASE message (elapsed 0ms, rc 0)
+2025-03-27 01:11:19.183091992 rotuer dhcp6c.wan.link.pppoe odhcp6c[615]: Failed to send RELEASE message to ff02::1:2 (Network unreachable)
+2025-03-27 01:11:19.183096392 rotuer dhcp6c.wan.link.pppoe odhcp6c[615]: (re)starting transaction on ppp0
+
+# dhcp6c noticed no ppp so tried to release address. it can't because
+#  ppp is down. how did it know it was time to say byebye?
+
+2025-03-27 01:11:19.198178337 rotuer int.link.a.192.168.8.1.dnsmasq dnsmasq-dhcp[631]: router advertisement on 2001:8b0:de3a:40de::, old prefix for int
+2025-03-27 01:11:19.199068668 rotuer dhcp6c.prefix.dhcp6c.wan.link.pppoe.int.link.a.192.168.8.1 Executed "ip address del 2001:8b0:de3a:40de::1/64 dev int", exit code true
+# our prefix delegation is no longer ours
+
+2025-03-27 01:11:19.199381632 rotuer dhcp6c.addr.dhcp6c.wan.link.pppoe.wan.link.pppoe ip: RTNETLINK answers: Address not available
+2025-03-27 01:11:19.199401552 rotuer dhcp6c.addr.dhcp6c.wan.link.pppoe.wan.link.pppoe /nix/store/anz85qjb2qw084f5xhpx4qndp260xr9i-lua-tty-aarch64-unknown-linux-musl/bin/lua: ...ch64-unknown-linux-musl-0.1/share/lua/5.3/anoia/init.lua:93: Error executing "ip address del 2001:8b0:1111:1111:0:ffff:d9a9:14de/128 dev ppp0" (exit)
+
+this is the ip6 address that ppp negotiated.  is this command failing because
+it was removed already?
+
+[....]
+2025-03-27 01:11:25.545493523 rotuer wan.link.pppoe Script /nix/store/9n2314z07z1g9dd3sqr0kndinjv0vma3-ip6-up started (pid 10346)
+2025-03-27 01:11:25.551376155 rotuer wan.link.pppoe Script /nix/store/9n2314z07z1g9dd3sqr0kndinjv0vma3-ip6-up finished (pid 10346), status = 0x0
+2025-03-27 01:11:25.552949055 rotuer wan.link.pppoe sh: write error: Broken pipe
+2025-03-27 01:11:25.553494501 rotuer wan.link.pppoe Script /nix/store/jkvnjhiqjahydmns33z5lni1hf9sl3x8-ip-up finished (pid 10345), status = 0x1
+2025-03-27 01:11:26.596566491 rotuer wlan0.link.hostapd wlan0: STA-OPMODE-SMPS-MODE-CHANGED 96:75:33:6c:6b:86 static
+2025-03-27 01:11:26.656620107 rotuer wlan0.link.hostapd wlan0: STA-OPMODE-SMPS-MODE-CHANGED 96:75:33:6c:6b:86 off
+2025-03-27 01:11:27.189590642 rotuer dhcp6c.wan.link.pppoe odhcp6c[615]: Failed to send RS (Network unreachable)
+
+it's supposed to send only one notification (each up-script only sends
+if an output the other has created is there).
+
+~~maybe we're not wiping the outputs when it terminates?~~ checked, we are.
+
+
+2025-03-27 01:11:27.279458704 rotuer int.link.a.192.168.8.1.dnsmasq dnsmasq-dhcp[631]: no address range available for DHCPv6 request via int
+2025-03-27 01:11:27.557594914 rotuer dhcp6c.wan.link.pppoe odhcp6c[615]: Failed to send SOLICIT message to ff02::1:2 (Network unreachable)
+2025-03-27 01:11:31.193583256 rotuer dhcp6c.wan.link.pppoe odhcp6c[615]: Failed to send RS (Network unreachable)
+
+Hypothesis
+
+1) when the remote end bounces ppp, s6 restarts the ppp process
+but this doesn't cause dependent services to notice, so odhcpc,
+which is bound to the interface, doesn't see that we now have a
+new interface but with the same name as the old one
+
+and/or
+
+2) there is a race between acquire-*.fnl starting and the
+prefix/address outptus being updated so that we miss the first
+value change.

examples/rotuer.nix

@@ -71,6 +71,7 @@ rec {
         interface = config.hardware.networkInterfaces.wan;
         username = secrets.l2tp.name;
         password = secrets.l2tp.password;
+        bandwidth = 70 * 1000 * 1000;
       };
       # once the wan has ipv4 connnectivity, should we run dhcp6
       # client to potentially get an address range ("prefix

modules/dhcp6c/acquire-delegated-prefix.fnl

@@ -25,6 +25,7 @@
 (fn run []
   (let [[state-directory lan-device] arg
         dir (svc.open state-directory)]
+    (update-prefixes lan-device [] (or (dir:output "prefix") []) system)
     (accumulate [addresses []
                  v (dir:events)]
       (update-prefixes lan-device addresses (or (v:output "prefix") []) system))))

modules/dhcp6c/acquire-wan-address.fnl

@@ -25,6 +25,7 @@
 (fn run []
   (let [[state-directory wan-device] arg
         dir (svc.open state-directory)]
+    (update-addresses wan-device [] (or (dir:output "address") []) system)
     (accumulate [addresses []
                  v (dir:events)]
       (update-addresses wan-device addresses (or (v:output "address") []) system))))

modules/dhcp6c/address.nix

@@ -10,7 +10,7 @@ let
 in
 longrun {
   inherit name;
-  run = "${script} $SERVICE_OUTPUTS/${client.name} $(output ${interface} ifname)";
+  run = "${script} ${client} $(output ${interface} ifname)";
   dependencies = [
     client
     interface

modules/dhcp6c/client.nix

@@ -2,19 +2,29 @@
   liminix,
   odhcp6c,
   odhcp-script,
+  svc
 }:
 { interface }:
 let
   inherit (liminix.services) longrun;
+  inherit (liminix) outputRef;
   name = "dhcp6c.${interface.name}";
-in
-longrun {
-  inherit name;
-  notification-fd = 10;
-  run = ''
-    export SERVICE_STATE=$SERVICE_OUTPUTS/${name}
-    ${odhcp6c}/bin/odhcp6c -s ${odhcp-script} -e -v -p /run/${name}.pid -P0 $(output ${interface} ifname)
-    )
-  '';
-  dependencies = [ interface ];
+  service =
+    longrun {
+      inherit name;
+      notification-fd = 10;
+      run = ''
+        export SERVICE_STATE=$SERVICE_OUTPUTS/${name}
+        ifname=$(output ${interface} ifname)
+        test -n "$ifname" && ${odhcp6c}/bin/odhcp6c -s ${odhcp-script} -e -v -p /run/${name}.pid -P0 $ifname
+        )
+      '';
+      dependencies = [ interface ];
+    };
+in svc.secrets.subscriber.build {
+  # if the ppp service gets restarted, the interface may be different and
+  # we will have to restart dhcp on the new one
+  watch = [ (outputRef interface "ifindex") ];
+  action = "restart";
+  inherit service;
 }

modules/dhcp6c/prefix.nix

@@ -10,7 +10,7 @@ let
 in
 longrun {
   inherit name;
-  run = "${script} $SERVICE_OUTPUTS/${client.name} $(output ${interface} ifname)";
+  run = "${script} ${client} $(output ${interface} ifname)";
   dependencies = [
     client
     interface

modules/firewall/service.nix

@@ -41,7 +41,7 @@ let
       '';
     };
 
-  rateHook =
+  rateHook6 =
     let rules =
           map
             (x: ''
@@ -50,7 +50,7 @@ let
                 local n = output(s, "ifname");
                 local bw = output(s, "bandwidth");
                 if n and bw then
-                  return "meta l4proto icmpv6 iifname ".. n .. " limit rate over " .. (math.floor (tonumber(bw) / 20)) .. " bytes/second drop"
+                  return "meta l4proto icmpv6 iifname ".. n .. " limit rate over " .. (math.floor (tonumber(bw) / 8 / 20)) .. " bytes/second drop"
                 else
                   return "# " .. (n or "not n") .. " " .. (bw or "not bw")
                 end
@@ -63,10 +63,35 @@ let
       inherit rules;
     };
 
+  rateHook4 =
+    let rules =
+          map
+            (x: ''
+               {{;
+                local s = "${x}";
+                local n = output(s, "ifname");
+                local bw = output(s, "bandwidth");
+                if n and bw then
+                  return "meta l4proto icmp iifname ".. n .. " limit rate over " .. (math.floor (tonumber(bw) / 8 / 20)) .. " bytes/second drop"
+                else
+                  return "# " .. (n or "not n") .. " " .. (bw or "not bw")
+                end
+               }}
+             '')
+            (concatLists (builtins.attrValues zones));
+    in {
+      type = "filter"; family = "ip";
+      hook = "input"; priority = "-1"; policy = "accept";
+      inherit rules;
+    };
+
   sets = (mapAttrs' (n: _: mkSet "ip" n) zones) //
          (mapAttrs' (n: _: mkSet "ip6" n) zones);
   allRules =
-    { icmp6-ratehook = rateHook; } //
+    {
+      icmp6-ratehook = rateHook6;
+      icmp4-ratehook = rateHook4;
+    } //
     (lib.recursiveUpdate
       extraRules
       (lib.recursiveUpdate sets rules));

modules/ppp/common.nix

@@ -13,6 +13,7 @@
   username,
   password,
   lcpEcho,
+  bandwidth,
   ppp-options,
   dependencies ? [ ],
 }:
@@ -35,18 +36,21 @@ let
     echo $3 > speed
     echo $4 > address
     echo $5 > peer-address
+    cat /sys/class/net/$1/ifindex > ifindex
     set +o nounset
     if test -n "''${DNS1}" ;then echo ''${DNS1} > ns1 ; fi
     if test -n "''${DNS2}" ;then echo ''${DNS2} > ns2 ; fi
-    test -e ipv6-address && echo >/proc/self/fd/10
+    touch ip-up
+    test -e ipv6-up && echo >/proc/self/fd/10
   '';
   ip6-up = writeAshScript "ip6-up" { } ''
     exec >&5 2>&5
     . ${serviceFns} 
     in_outputs ${name}
-    echo $4 > ipv6-address
     echo $5 > ipv6-peer-address
-    test -e ifname && echo >/proc/self/fd/10
+    echo $4 > ipv6-address
+    touch ipv6-up
+    test -e ip-up && echo >/proc/self/fd/10
   '';
   literal_or_output =
     let
@@ -115,7 +119,7 @@ let
       ${command}
     '';
     notification-fd = 10;
-#    properties.bandwidth = 3 * 1000 * 1000;
+    properties.bandwidth = bandwidth;
     timeout-up =
       if lcpEcho.failure != null then (10 + lcpEcho.failure * lcpEcho.interval) * 1000 else 60 * 1000;
     inherit dependencies;

modules/ppp/default.nix

@@ -56,6 +56,11 @@ in
         default = null;
         description = "password";
       };
+      bandwidth = mkOption {
+        type = types.nullOr (types.int);
+        default = null;
+        description = "approximate bandwidth in bytes/second. Used to calculate rate limits for ICMP";
+      };
       lcpEcho = {
         adaptive = mkOption {
           description = "send LCP echo-request frames only if no traffic was received from the peer since the last echo-request was sent";

modules/ppp/l2tp.nix

@@ -13,6 +13,7 @@
   lns,
   ppp-options,
   lcpEcho,
+  bandwidth,
   username,
   password,
   debug,
@@ -40,6 +41,7 @@ common {
     username
     password
     lcpEcho
+    bandwidth
     ppp-options
     ;
   command = ''

modules/ppp/pppoe.nix

@@ -13,6 +13,7 @@
   interface,
   ppp-options,
   lcpEcho,
+  bandwidth,
   username,
   password,
   debug,
@@ -31,6 +32,7 @@ common {
     username
     password
     lcpEcho
+    bandwidth
     ppp-options
     ;
   command = ''

modules/secrets/subscriber.nix

@@ -5,6 +5,7 @@
   s6,
   s6-rc,
   watch-outputs,
+  s6-rc-up-tree
 }:
 {
   watch,
@@ -47,7 +48,7 @@ let
         echo waiting for $dir
         if test -e $dir/notification-fd; then flag="-U"; else flag="-u"; fi
         ${s6}/bin/s6-svwait $flag /run/service/${name} || exit
-        PATH=${s6-rc}/bin:${s6}/bin:$PATH
+        PATH=${s6-rc}/bin:${s6}/bin:${s6-rc-up-tree}/bin:$PATH
         ${watch-outputs}/bin/watch-outputs ${restart-flag} ${name} ${refs}
       '';
     };

pkgs/watch-outputs/watch-outputs.fnl

@@ -54,7 +54,7 @@
 (fn do-action [action service]
   (case action
     :restart (system (%% "s6-svc -r /run/service/%s" service))
-    :restart-all (system (%% "s6-rc -b -d %q; s6-rc-up-tree %q" service service))
+    :restart-all (system (%% "s6-rc -b -d change %q; s6-rc-up-tree %q" service service))
     [:signal n] (system (%% "s6-svc -s %q /run/service/%s" n service))))
 
 (local POLLIN  0x0001)