remove unused function

* the lua necessary is quite wordy, but it's less of a hack than
post-processing the rules file with pseudo-sed to get rid of `elements
= { }` lines

* also switch from stop/starting the firewall service to using a
signal, so that we don't go briefly offline every time a new interface
appears

THOUGHTS.txt

@@ -7109,3 +7109,90 @@ commit until I have some way to see if they're working. the pppoe test
 will check both firewall zones so _should_ start to fail with the
 current watch-outputs (because only one service) and then pass when we
 put the new one in
+
+Fri Feb 28 01:00:03 GMT 2025
+
+Well, it works at least well enough to pass the test. There is an awful hack
+though, because nftables doesn't accept "elements = { }" as valid syntax
+for a set with no elements, so we post-process the file to wipe those lines
+
+I wonder if we could instead create the set empty and then use the "other"
+nftables format to generate commands that add the elements. If it's
+all in the same file (or included files) it will continue to be atomic
+
+Other options
+
+- is the nftables json format any better? we will have to rebuild it
+  with json support, may be bugger
+- write lua bindings to libnftables
+
+Fri Feb 28 23:31:06 GMT 2025
+
+adding json would add 76 + 88k to the image, but I think it would also
+mean we have to rewrite all the default rules in json format
+
+
+
+with json
+[dan@loaclhost:~/src/liminix]$ du  result/
+20      result/share/doc/nftables/examples
+24      result/share/doc/nftables
+28      result/share/doc
+12      result/share/man/man3
+16      result/share/man/man5
+44      result/share/man/man8
+76      result/share/man
+60      result/share/nftables
+168     result/share
+36      result/etc/nftables/osf
+40      result/etc/nftables
+44      result/etc
+76      result/bin
+8       result/include/nftables
+12      result/include
+8       result/lib/pkgconfig
+1172    result/lib
+1476    result/
+
+[dan@loaclhost:~/src/liminix]$ du /nix/store/l0zsvldsskiv52b4c9b21ziq5z1qr7vn-jansson-mips-unknown-linux-musl-2.14/
+84      /nix/store/l0zsvldsskiv52b4c9b21ziq5z1qr7vn-jansson-mips-unknown-linux-musl-2.14/lib
+88      /nix/store/l0zsvldsskiv52b4c9b21ziq5z1qr7vn-jansson-mips-unknown-linux-musl-2.14/
+
+without:
+[dan@loaclhost:~/src/liminix]$ du  result/
+20      result/share/doc/nftables/examples
+24      result/share/doc/nftables
+28      result/share/doc
+12      result/share/man/man3
+16      result/share/man/man5
+44      result/share/man/man8
+76      result/share/man
+60      result/share/nftables
+168     result/share
+36      result/etc/nftables/osf
+40      result/etc/nftables
+44      result/etc
+76      result/bin
+8       result/include/nftables
+12      result/include
+8       result/lib/pkgconfig
+1096    result/lib
+1400    result/
+
+
+Sat Mar  1 23:43:17 GMT 2025
+
+I don't think json is going to help because either we'd have to do
+
+ elements = map (f: "{{ lookup(f, \"ifname\") }}") zones.${zone}
+
+and there would  be null elements in the places for the interfaces that don't exist
+yet, or we'd have to write actual json syntax at runtime, at which point why don't
+we write the trad nftables syntax instead?
+
+let's write a firewall .nftables file consisting of the zone set
+elements plus an "include" directive for the rest of the firewall. NOTE THAT
+we may still need to template the rest of the firewall if we want to have
+other variables (rate limits) in it, because the rules for that need to be
+inserted ahead of the rules for accepting icmp, and there's no way to
+do that without

modules/firewall/service.nix

@@ -21,7 +21,7 @@ let
   inherit (lib.attrsets) mapAttrs' nameValuePair mapAttrsToList;
   inherit (lib.strings) concatStringsSep;
   inherit (lib.lists) flatten;
-  inherit (builtins) concatLists attrValues;
+  inherit (builtins) concatLists toJSON attrValues;
   inherit (liminix) outputRef;
   mkSet =
     family: name:
@@ -29,38 +29,50 @@ let
       kind = "set";
       inherit name family;
       type = "ifname";
-      elements = map (s: "{{ output(${builtins.toJSON s}, \"ifname\", \"\") }}") zones.${name};
-    };
+      extraText = ''
+      {{;
+         local services = { ${concatStringsSep ", " (map toJSON zones.${name})} }
+         local ifnames = {}
+         for _, v in ipairs(services) do
+           local o = output(v, "ifname")
+           if o then table.insert(ifnames, o) end
+         end
+         if (#ifnames > 0) then
+           return "elements = { " .. table.concat(ifnames, ", ") .. " }\n"
+         else
+           return ""
+         end
+      }}
+      '';
+  };
   sets = (mapAttrs' (n: _: mkSet "ip" n) zones) //
          (mapAttrs' (n: _: mkSet "ip6" n) zones);
-  allRules = lib.recursiveUpdate extraRules (lib.recursiveUpdate (builtins.trace sets sets) rules);
+  allRules = lib.recursiveUpdate extraRules (lib.recursiveUpdate sets rules);
   script = firewallgen "firewall1.nft" allRules;
-  ifwatch = writeFennel "ifwatch" {
-    packages = [
-      anoia
-      lualinux
-      linotify
-    ];
-    mainFunction = "run";
-  } ./ifwatch.fnl;
   watchArg = z: intfs: map (i: "${z}:${i}/.outputs") intfs;
   name = "firewall";
   service = longrun {
     inherit name;
     run = ''
-      mkdir -p /run/${name}; in_outputs ${name}
-      # exec > /dev/console 2>&1
-      echo RESTARTING FIREWALL >/dev/console
       PATH=${nftables}/bin:${lua}/bin:$PATH
-      ${output-template}/bin/output-template '{{' '}}' < ${script} | lua -e 'for x in io.lines() do if not string.match(x, "elements = {%s+}") then print(x) end; end'   > /run/${name}/fw.nft
-      # cat /run/${name}/fw.nft > /dev/console
-      nft -f /run/${name}/fw.nft
-      while sleep 86400 ; do : ; done
+      reload() {
+         echo reloading firewall
+         ${output-template}/bin/output-template '{{' '}}' < ${script}  > /run/${name}/fw.nft;
+         nft -f /run/${name}/fw.nft ;
+      }
+      trap reload SIGUSR1
+      mkdir -p /run/${name}; in_outputs ${name}
+      reload
+      while :; do
+        # signals sent to ash won't interrupt sleep, but will interrupt wait
+        sleep 86400 & wait
+      done
     '';
     finish = "${nftables}/bin/nft flush ruleset";
   };
 in
 svc.secrets.subscriber.build {
+  action = "usr1";
   watch =
     concatLists
       (mapAttrsToList (_zone : services : map (s: outputRef s "ifname") services) zones);

modules/secrets/subscriber.nix

@@ -28,9 +28,9 @@ let
       "quit" = "-s 3";
       "kill" = "-s 9";
       "term" = "-s 15";
-      "winch" = "-s 28";
-      "usr1" = "-s 10";
-      "usr2" = "-s 12";
+      "winch" = "-s 20";
+      "usr1" = "-s usr1";
+      "usr2" = "-s usr2";
     }
     .${action};
 
@@ -56,8 +56,5 @@ service.overrideAttrs (o: {
   buildInputs = (lim.orEmpty o.buildInputs) ++ optional (watch != []) watcher;
   dependencies =
     (lim.orEmpty o.dependencies)
-    # ++ optionals
-    #   (watch != [])
-      #   ([ watcher ] ++ watched-services);
-      ;
+    ++ optional (watch != []) watcher;
 })

pkgs/firewallgen/default.nix

@@ -10,6 +10,7 @@ let
     splitString
     hasInfix
     substring
+    optionalString
     ;
   inherit (lib.lists) groupBy;
   inherit (lib.attrsets) mapAttrsToList;
@@ -56,12 +57,14 @@ let
       name,
       type,
       elements ? [ ],
+      extraText ? null,
       ...
     }:
     ''
       set ${name}  {
         type ${type}
         ${if elements != [ ] then "elements = { ${concatStringsSep ", " (builtins.trace elements elements)} }" else ""}
+        ${optionalString (extraText != null) extraText}
       }
     '';
 

pkgs/output-template/Makefile

@@ -1,3 +0,0 @@
-check:
-	./output-template  '{{' '}}' < example.ini > output
-	diff -u output example.ini.expected

pkgs/output-template/default.nix

@@ -2,6 +2,7 @@
   fetchurl,
   writeFennel,
   fennel,
+  fennelrepl,
   runCommand,
   lua,
   anoia,
@@ -17,9 +18,9 @@ stdenv.mkDerivation {
   src = ./.;
 
   buildInputs = [ lua ];
-  doCheck = true;
-
+  nativeBuildInputs = [ fennelrepl ] ;
   buildPhase = ''
+    fennelrepl --test ./output-template.fnl
     cp -p ${
       writeFennel name {
         packages = [
@@ -27,11 +28,11 @@ stdenv.mkDerivation {
           lualinux
           linotify
         ];
+        macros = [ anoia.dev ];
         mainFunction = "run";
       } ./output-template.fnl
     } ${name}
   '';
-  checkPhase = "make check";
   installPhase = ''
     install -D ${name} $out/bin/${name}
   '';

pkgs/output-template/output-template.fnl

@@ -24,21 +24,43 @@
   (let [delim (.. opening "(.-)" closing)
         myenv {
                : string
+               : table
+               : ipairs
                :output
                (fn [service-path path default]
                  (let [s (assert (svc.open (.. service-path "/.outputs")))]
                    (or (s:output path) default)))
-               :lua_quote #(string.format "%q" %1)
+               :lua_quote #(string.format "%q" $1)
                :json_quote (fn [x] (.. "\"" (json-escape x) "\""))
                }]
     (string.gsub text delim
                  (fn [x]
-                   (assert ((load (.. "return " x) x :t myenv))
-                           (string.format "missing value for %q" x))))))
+                   (let [chunk (if (= (x:sub 1 1) ";")
+                                   (x:sub 2)
+                                   (.. "return " x))]
+                     (assert ((load chunk x :t myenv))
+                             (string.format "missing value for %q" x)))))))
 
 (fn run []
   (let [[opening closing] arg
         out (substitute (: (io.input) :read "*a") opening closing)]
     (io.write out)))
 
+(import-macros { : define-tests : expect : expect= } :anoia.assert)
+(define-tests
+  (expect= (pick-values 1 (substitute "var={{ 2 + 3 }}" "{{" "}}")) "var=5")
+  (expect= (pick-values 1 (substitute "{{ json_quote(\"o'reilly\") }}" "{{" "}}"))
+           "\"o'reilly\"")
+
+  (expect= (pick-values 1 (substitute "{{; local a=9; return a }}" "{{" "}}")) "9")
+
+  ;; "globals" set in one interpolation are available in subsequent ones
+  (expect= (pick-values 1 (substitute "{{; a=42; return a }} {{ a and 999 or 0 }}" "{{" "}}")) "42 999")
+
+  (fn slurp [name]
+    (with-open [f (assert (io.open name))] (f:read "*a")))
+  (expect=
+   (pick-values 1 (substitute (slurp "example.ini")  "{{" "}}"))
+   (slurp "example.ini.expected")))
+
 { : run }

pkgs/watch-outputs/Makefile

@@ -1,3 +0,0 @@
-check:
-	fennelrepl ./test.fnl  '{{' '}}' < example.ini > output
-	diff -u output example.ini.expected

pkgs/watch-outputs/default.nix

@@ -34,7 +34,6 @@ stdenv.mkDerivation {
         mainFunction = "run";
       } ./watch-outputs.fnl
     } ${name}
-    make check
   '';
 
   installPhase = ''

pkgs/watch-outputs/example-service/.outputs/addresses/1/attribute

@@ -1 +0,0 @@
-a11

pkgs/watch-outputs/example-service/.outputs/addresses/3/attribute

@@ -1 +0,0 @@
-a33

pkgs/watch-outputs/example-service/.outputs/addresses/5/attribute

@@ -1 +0,0 @@
-a55

pkgs/watch-outputs/example-service/.outputs/addresses/6/attribute

@@ -1 +0,0 @@
-a66

pkgs/watch-outputs/example-service/.outputs/colours/black

@@ -1 +0,0 @@
-000000

pkgs/watch-outputs/example-service/.outputs/colours/blue

@@ -1 +0,0 @@
-0000ff

pkgs/watch-outputs/example-service/.outputs/colours/green

@@ -1 +0,0 @@
-00ff00

pkgs/watch-outputs/example-service/.outputs/colours/red

@@ -1 +0,0 @@
-ff0000

pkgs/watch-outputs/example-service/.outputs/name

@@ -1 +0,0 @@
-eth1

pkgs/watch-outputs/example.ini

@@ -1,3 +0,0 @@
-wpa_passphrase={{ output("./example-service","colours/black")  }}
-think = {{ string.format("%q", output("./example-service","colours/blue")) }}
-argonaut = {{ json_quote "hello\ngoodbye\tnext\027" }}

pkgs/watch-outputs/example.ini.expected

@@ -1,3 +0,0 @@
-wpa_passphrase=000000
-think = "0000ff"
-argonaut = "hello\ngoodbye\tnext\u001B"

pkgs/watch-outputs/output-template.fnl

@@ -1,44 +0,0 @@
-(local svc (require :anoia.svc))
-
-(fn json-escape [s]
-  ;; All Unicode characters may be placed within the quotation marks,
-  ;; except for the characters that MUST be escaped:
-  ;; quotation mark, reverse solidus, and the control characters (U+0000
-  ;; through U+001F). (RFC 8259)
-  (-> s
-      (string.gsub
-       "[\"\b\f\n\r\t]" {
-                         "\b" "\\b"
-                         "\"" "\\\""
-                         "\f" "\\f"
-                         "\n" "\\n"
-                         "\r" "\\r"
-                         "\t" "\\t"
-                         })
-      (string.gsub
-       "([\x00-\x1b])"
-       (fn [x] (string.format "\\u%04X" (string.byte x))))))
-
-
-(fn substitute [text opening closing]
-  (let [delim (.. opening "(.-)" closing)
-        myenv {
-               : string
-               :output
-               (fn [service-path path]
-                 (let [s (assert (svc.open (.. service-path "/.outputs")))]
-                   (s:output path)))
-               :lua_quote #(string.format "%q" $1)
-               :json_quote (fn [x] (.. "\"" (json-escape x) "\""))
-               }]
-    (string.gsub text delim
-                 (fn [x]
-                   (assert ((load (.. "return " x) x :t myenv))
-                           (string.format "missing value for %q" x))))))
-
-(fn run []
-  (let [[opening closing] arg
-        out (substitute (: (io.input) :read "*a") opening closing)]
-    (io.write out)))
-
-{ : run }

pkgs/watch-outputs/watch-outputs.fnl

@@ -58,7 +58,7 @@
   (case action
     :restart (system (%% "s6-svc -r /run/service/%s" service))
     :restart-all (system (%% "s6-rc -b -d %q; s6-rc-up-tree %q" service service))
-    [:signal n] (system (%% "s6-svc -s %d /run/service/%s" n service))))
+    [:signal n] (system (%% "s6-svc -s %q /run/service/%s" n service))))
 
 (local POLLIN  0x0001)
 (local POLLHUP 0x0010)
@@ -71,7 +71,7 @@
 
 (fn open-services [output-references]
   (collect [s p (pairs output-references)]
-    (values (assert (svc.open (.. s "/.outputs"))) p)))
+    (values (svc.open (.. s "/.outputs")) p)))
 
 (fn run []
   (let [trees {}
@@ -89,7 +89,6 @@
         (each [service paths (pairs services)]
           (let [new-tree (service:output ".")]
             (when (changed? paths (. trees service) new-tree)
-              (print "watched path event:"  action controlled-service)
               (do-action action controlled-service))))))))
 
 

tests/pppoe/getaddress.expect

@@ -8,29 +8,39 @@ send "PS1=\$(echo 'I1JFQURZIyA=' | base64 -d); stty -echo\n"
 expect "#READY#"
 set FINISHED 0
 set EXIT "1" 
-while { $FINISHED < 10 } {
+while { $FINISHED < 20 } {
   send "ip address show dev ppp0 | grep ppp0\n"
   expect {
-    "192.168.100.1" { set FINISHED 20; set EXIT 0; }
+    "192.168.100.1" { set FINISHED 200; set EXIT 0; }
     "can't find device" { send_user "waiting ..." ; sleep 3 }
     "DOWN" { send_user "waiting ..." ; sleep 3 }
   }
   set FINISHED [ expr $FINISHED + 1 ]
 }
 expect "#READY#"
-send "nft list set ip table-ip wan || touch /non/existent\n"
-expect {
-  "ppp0" { puts "ppp0 found " }
-  "{ }" { puts "missing ifname"; exit 1 }
-  "No such file or directory" { exit 1 }
-}  
+send "s6-svwait -U /run/service/wan.link.pppoe\n"
 expect "#READY#"
 
+set timeout 30
 send "nft list set ip table-ip lan || touch /non/existent\n"
 expect {
   "lan" { puts "lan found" }
   "{ }" { puts "missing ifname"; exit 1 }
   "No such file or directory" { exit 1 }
 }  
+
 expect "#READY#"
+
+# if the test fails for no apparent reason, it is quite likely because
+# the firewall hasn't had time to reload after the new interface
+# appears and you just have to make this sleep longer. Ew, yes
+send "sleep 10; nft list set ip table-ip wan || touch /non/existent\n"
+expect {
+  "ppp0" { puts "ppp0 found " }
+  "{ }" { puts "missing ifname"; exit 1 }
+  "No such file or directory" { exit 1 }
+  timeout { exit 1 }
+}  
+expect "#READY#"
+
 exit $EXIT