THOUGHTS.txt

@@ -7109,90 +7109,3 @@ commit until I have some way to see if they're working. the pppoe test
 will check both firewall zones so _should_ start to fail with the
 current watch-outputs (because only one service) and then pass when we
 put the new one in
-
-Fri Feb 28 01:00:03 GMT 2025
-
-Well, it works at least well enough to pass the test. There is an awful hack
-though, because nftables doesn't accept "elements = { }" as valid syntax
-for a set with no elements, so we post-process the file to wipe those lines
-
-I wonder if we could instead create the set empty and then use the "other"
-nftables format to generate commands that add the elements. If it's
-all in the same file (or included files) it will continue to be atomic
-
-Other options
-
-- is the nftables json format any better? we will have to rebuild it
-  with json support, may be bugger
-- write lua bindings to libnftables
-
-Fri Feb 28 23:31:06 GMT 2025
-
-adding json would add 76 + 88k to the image, but I think it would also
-mean we have to rewrite all the default rules in json format
-
-
-
-with json
-[dan@loaclhost:~/src/liminix]$ du  result/
-20      result/share/doc/nftables/examples
-24      result/share/doc/nftables
-28      result/share/doc
-12      result/share/man/man3
-16      result/share/man/man5
-44      result/share/man/man8
-76      result/share/man
-60      result/share/nftables
-168     result/share
-36      result/etc/nftables/osf
-40      result/etc/nftables
-44      result/etc
-76      result/bin
-8       result/include/nftables
-12      result/include
-8       result/lib/pkgconfig
-1172    result/lib
-1476    result/
-
-[dan@loaclhost:~/src/liminix]$ du /nix/store/l0zsvldsskiv52b4c9b21ziq5z1qr7vn-jansson-mips-unknown-linux-musl-2.14/
-84      /nix/store/l0zsvldsskiv52b4c9b21ziq5z1qr7vn-jansson-mips-unknown-linux-musl-2.14/lib
-88      /nix/store/l0zsvldsskiv52b4c9b21ziq5z1qr7vn-jansson-mips-unknown-linux-musl-2.14/
-
-without:
-[dan@loaclhost:~/src/liminix]$ du  result/
-20      result/share/doc/nftables/examples
-24      result/share/doc/nftables
-28      result/share/doc
-12      result/share/man/man3
-16      result/share/man/man5
-44      result/share/man/man8
-76      result/share/man
-60      result/share/nftables
-168     result/share
-36      result/etc/nftables/osf
-40      result/etc/nftables
-44      result/etc
-76      result/bin
-8       result/include/nftables
-12      result/include
-8       result/lib/pkgconfig
-1096    result/lib
-1400    result/
-
-
-Sat Mar  1 23:43:17 GMT 2025
-
-I don't think json is going to help because either we'd have to do
-
- elements = map (f: "{{ lookup(f, \"ifname\") }}") zones.${zone}
-
-and there would  be null elements in the places for the interfaces that don't exist
-yet, or we'd have to write actual json syntax at runtime, at which point why don't
-we write the trad nftables syntax instead?
-
-let's write a firewall .nftables file consisting of the zone set
-elements plus an "include" directive for the rest of the firewall. NOTE THAT
-we may still need to template the rest of the firewall if we want to have
-other variables (rate limits) in it, because the rules for that need to be
-inserted ahead of the rules for accepting icmp, and there's no way to
-do that without

modules/firewall/service.nix

@@ -21,7 +21,7 @@ let
   inherit (lib.attrsets) mapAttrs' nameValuePair mapAttrsToList;
   inherit (lib.strings) concatStringsSep;
   inherit (lib.lists) flatten;
-  inherit (builtins) concatLists toJSON attrValues;
+  inherit (builtins) concatLists attrValues;
   inherit (liminix) outputRef;
   mkSet =
     family: name:
@@ -29,50 +29,38 @@ let
       kind = "set";
       inherit name family;
       type = "ifname";
-      extraText = ''
+      elements = map (s: "{{ output(${builtins.toJSON s}, \"ifname\", \"\") }}") zones.${name};
-      {{;
+    };
-         local services = { ${concatStringsSep ", " (map toJSON zones.${name})} }
-         local ifnames = {}
-         for _, v in ipairs(services) do
-           local o = output(v, "ifname")
-           if o then table.insert(ifnames, o) end
-         end
-         if (#ifnames > 0) then
-           return "elements = { " .. table.concat(ifnames, ", ") .. " }\n"
-         else
-           return ""
-         end
-      }}
-      '';
-  };
   sets = (mapAttrs' (n: _: mkSet "ip" n) zones) //
          (mapAttrs' (n: _: mkSet "ip6" n) zones);
-  allRules = lib.recursiveUpdate extraRules (lib.recursiveUpdate sets rules);
+  allRules = lib.recursiveUpdate extraRules (lib.recursiveUpdate (builtins.trace sets sets) rules);
   script = firewallgen "firewall1.nft" allRules;
+  ifwatch = writeFennel "ifwatch" {
+    packages = [
+      anoia
+      lualinux
+      linotify
+    ];
+    mainFunction = "run";
+  } ./ifwatch.fnl;
   watchArg = z: intfs: map (i: "${z}:${i}/.outputs") intfs;
   name = "firewall";
   service = longrun {
     inherit name;
     run = ''
-      PATH=${nftables}/bin:${lua}/bin:$PATH
-      reload() {
-         echo reloading firewall
-         ${output-template}/bin/output-template '{{' '}}' < ${script}  > /run/${name}/fw.nft;
-         nft -f /run/${name}/fw.nft ;
-      }
-      trap reload SIGUSR1
       mkdir -p /run/${name}; in_outputs ${name}
-      reload
+      # exec > /dev/console 2>&1
-      while :; do
+      echo RESTARTING FIREWALL >/dev/console
-        # signals sent to ash won't interrupt sleep, but will interrupt wait
+      PATH=${nftables}/bin:${lua}/bin:$PATH
-        sleep 86400 & wait
+      ${output-template}/bin/output-template '{{' '}}' < ${script} | lua -e 'for x in io.lines() do if not string.match(x, "elements = {%s+}") then print(x) end; end'   > /run/${name}/fw.nft
-      done
+      # cat /run/${name}/fw.nft > /dev/console
+      nft -f /run/${name}/fw.nft
+      while sleep 86400 ; do : ; done
     '';
     finish = "${nftables}/bin/nft flush ruleset";
   };
 in
 svc.secrets.subscriber.build {
-  action = "usr1";
   watch =
     concatLists
       (mapAttrsToList (_zone : services : map (s: outputRef s "ifname") services) zones);

modules/secrets/subscriber.nix

@@ -28,9 +28,9 @@ let
       "quit" = "-s 3";
       "kill" = "-s 9";
       "term" = "-s 15";
-      "winch" = "-s 20";
+      "winch" = "-s 28";
-      "usr1" = "-s usr1";
+      "usr1" = "-s 10";
-      "usr2" = "-s usr2";
+      "usr2" = "-s 12";
     }
     .${action};
 
@@ -56,5 +56,8 @@ service.overrideAttrs (o: {
   buildInputs = (lim.orEmpty o.buildInputs) ++ optional (watch != []) watcher;
   dependencies =
     (lim.orEmpty o.dependencies)
-    ++ optional (watch != []) watcher;
+    # ++ optionals
+    #   (watch != [])
+      #   ([ watcher ] ++ watched-services);
+      ;
 })

pkgs/firewallgen/default.nix

@@ -10,7 +10,6 @@ let
     splitString
     hasInfix
     substring
-    optionalString
     ;
   inherit (lib.lists) groupBy;
   inherit (lib.attrsets) mapAttrsToList;
@@ -57,14 +56,12 @@ let
       name,
       type,
       elements ? [ ],
-      extraText ? null,
       ...
     }:
     ''
       set ${name}  {
         type ${type}
         ${if elements != [ ] then "elements = { ${concatStringsSep ", " (builtins.trace elements elements)} }" else ""}
-        ${optionalString (extraText != null) extraText}
       }
     '';
 

pkgs/output-template/Makefile

@@ -0,0 +1,3 @@
+check:
+	./output-template  '{{' '}}' < example.ini > output
+	diff -u output example.ini.expected

pkgs/output-template/default.nix

@@ -2,7 +2,6 @@
   fetchurl,
   writeFennel,
   fennel,
-  fennelrepl,
   runCommand,
   lua,
   anoia,
@@ -18,9 +17,9 @@ stdenv.mkDerivation {
   src = ./.;
 
   buildInputs = [ lua ];
-  nativeBuildInputs = [ fennelrepl ] ;
+  doCheck = true;
+
   buildPhase = ''
-    fennelrepl --test ./output-template.fnl
     cp -p ${
       writeFennel name {
         packages = [
@@ -28,11 +27,11 @@ stdenv.mkDerivation {
           lualinux
           linotify
         ];
-        macros = [ anoia.dev ];
         mainFunction = "run";
       } ./output-template.fnl
     } ${name}
   '';
+  checkPhase = "make check";
   installPhase = ''
     install -D ${name} $out/bin/${name}
   '';

pkgs/output-template/output-template.fnl

@@ -24,43 +24,21 @@
   (let [delim (.. opening "(.-)" closing)
         myenv {
                : string
-               : table
-               : ipairs
                :output
                (fn [service-path path default]
                  (let [s (assert (svc.open (.. service-path "/.outputs")))]
                    (or (s:output path) default)))
-               :lua_quote #(string.format "%q" $1)
+               :lua_quote #(string.format "%q" %1)
                :json_quote (fn [x] (.. "\"" (json-escape x) "\""))
                }]
     (string.gsub text delim
                  (fn [x]
-                   (let [chunk (if (= (x:sub 1 1) ";")
+                   (assert ((load (.. "return " x) x :t myenv))
-                                   (x:sub 2)
+                           (string.format "missing value for %q" x))))))
-                                   (.. "return " x))]
-                     (assert ((load chunk x :t myenv))
-                             (string.format "missing value for %q" x)))))))
 
 (fn run []
   (let [[opening closing] arg
         out (substitute (: (io.input) :read "*a") opening closing)]
     (io.write out)))
 
-(import-macros { : define-tests : expect : expect= } :anoia.assert)
-(define-tests
-  (expect= (pick-values 1 (substitute "var={{ 2 + 3 }}" "{{" "}}")) "var=5")
-  (expect= (pick-values 1 (substitute "{{ json_quote(\"o'reilly\") }}" "{{" "}}"))
-           "\"o'reilly\"")
-
-  (expect= (pick-values 1 (substitute "{{; local a=9; return a }}" "{{" "}}")) "9")
-
-  ;; "globals" set in one interpolation are available in subsequent ones
-  (expect= (pick-values 1 (substitute "{{; a=42; return a }} {{ a and 999 or 0 }}" "{{" "}}")) "42 999")
-
-  (fn slurp [name]
-    (with-open [f (assert (io.open name))] (f:read "*a")))
-  (expect=
-   (pick-values 1 (substitute (slurp "example.ini")  "{{" "}}"))
-   (slurp "example.ini.expected")))
-
 { : run }

pkgs/watch-outputs/Makefile

@@ -0,0 +1,3 @@
+check:
+	fennelrepl ./test.fnl  '{{' '}}' < example.ini > output
+	diff -u output example.ini.expected

pkgs/watch-outputs/default.nix

@@ -34,6 +34,7 @@ stdenv.mkDerivation {
         mainFunction = "run";
       } ./watch-outputs.fnl
     } ${name}
+    make check
   '';
 
   installPhase = ''

pkgs/watch-outputs/example-service/.outputs/addresses/1/attribute

@@ -0,0 +1 @@
+a11

pkgs/watch-outputs/example-service/.outputs/addresses/3/attribute

@@ -0,0 +1 @@
+a33

pkgs/watch-outputs/example-service/.outputs/addresses/5/attribute

@@ -0,0 +1 @@
+a55

pkgs/watch-outputs/example-service/.outputs/addresses/6/attribute

@@ -0,0 +1 @@
+a66

pkgs/watch-outputs/example-service/.outputs/colours/black

@@ -0,0 +1 @@
+000000

pkgs/watch-outputs/example-service/.outputs/colours/blue

@@ -0,0 +1 @@
+0000ff

pkgs/watch-outputs/example-service/.outputs/colours/green

@@ -0,0 +1 @@
+00ff00

pkgs/watch-outputs/example-service/.outputs/colours/red

@@ -0,0 +1 @@
+ff0000

pkgs/watch-outputs/example-service/.outputs/name

@@ -0,0 +1 @@
+eth1

pkgs/watch-outputs/example.ini

@@ -0,0 +1,3 @@
+wpa_passphrase={{ output("./example-service","colours/black")  }}
+think = {{ string.format("%q", output("./example-service","colours/blue")) }}
+argonaut = {{ json_quote "hello\ngoodbye\tnext\027" }}

pkgs/watch-outputs/example.ini.expected

@@ -0,0 +1,3 @@
+wpa_passphrase=000000
+think = "0000ff"
+argonaut = "hello\ngoodbye\tnext\u001B"

pkgs/watch-outputs/output-template.fnl

@@ -0,0 +1,44 @@
+(local svc (require :anoia.svc))
+
+(fn json-escape [s]
+  ;; All Unicode characters may be placed within the quotation marks,
+  ;; except for the characters that MUST be escaped:
+  ;; quotation mark, reverse solidus, and the control characters (U+0000
+  ;; through U+001F). (RFC 8259)
+  (-> s
+      (string.gsub
+       "[\"\b\f\n\r\t]" {
+                         "\b" "\\b"
+                         "\"" "\\\""
+                         "\f" "\\f"
+                         "\n" "\\n"
+                         "\r" "\\r"
+                         "\t" "\\t"
+                         })
+      (string.gsub
+       "([\x00-\x1b])"
+       (fn [x] (string.format "\\u%04X" (string.byte x))))))
+
+
+(fn substitute [text opening closing]
+  (let [delim (.. opening "(.-)" closing)
+        myenv {
+               : string
+               :output
+               (fn [service-path path]
+                 (let [s (assert (svc.open (.. service-path "/.outputs")))]
+                   (s:output path)))
+               :lua_quote #(string.format "%q" $1)
+               :json_quote (fn [x] (.. "\"" (json-escape x) "\""))
+               }]
+    (string.gsub text delim
+                 (fn [x]
+                   (assert ((load (.. "return " x) x :t myenv))
+                           (string.format "missing value for %q" x))))))
+
+(fn run []
+  (let [[opening closing] arg
+        out (substitute (: (io.input) :read "*a") opening closing)]
+    (io.write out)))
+
+{ : run }

pkgs/watch-outputs/watch-outputs.fnl

@@ -58,7 +58,7 @@
   (case action
     :restart (system (%% "s6-svc -r /run/service/%s" service))
     :restart-all (system (%% "s6-rc -b -d %q; s6-rc-up-tree %q" service service))
-    [:signal n] (system (%% "s6-svc -s %q /run/service/%s" n service))))
+    [:signal n] (system (%% "s6-svc -s %d /run/service/%s" n service))))
 
 (local POLLIN  0x0001)
 (local POLLHUP 0x0010)
@@ -71,7 +71,7 @@
 
 (fn open-services [output-references]
   (collect [s p (pairs output-references)]
-    (values (svc.open (.. s "/.outputs")) p)))
+    (values (assert (svc.open (.. s "/.outputs"))) p)))
 
 (fn run []
   (let [trees {}
@@ -89,6 +89,7 @@
         (each [service paths (pairs services)]
           (let [new-tree (service:output ".")]
             (when (changed? paths (. trees service) new-tree)
+              (print "watched path event:"  action controlled-service)
               (do-action action controlled-service))))))))
 
 

tests/pppoe/getaddress.expect

@@ -8,39 +8,29 @@ send "PS1=\$(echo 'I1JFQURZIyA=' | base64 -d); stty -echo\n"
 expect "#READY#"
 set FINISHED 0
 set EXIT "1" 
-while { $FINISHED < 20 } {
+while { $FINISHED < 10 } {
   send "ip address show dev ppp0 | grep ppp0\n"
   expect {
-    "192.168.100.1" { set FINISHED 200; set EXIT 0; }
+    "192.168.100.1" { set FINISHED 20; set EXIT 0; }
     "can't find device" { send_user "waiting ..." ; sleep 3 }
     "DOWN" { send_user "waiting ..." ; sleep 3 }
   }
   set FINISHED [ expr $FINISHED + 1 ]
 }
 expect "#READY#"
-send "s6-svwait -U /run/service/wan.link.pppoe\n"
+send "nft list set ip table-ip wan || touch /non/existent\n"
+expect {
+  "ppp0" { puts "ppp0 found " }
+  "{ }" { puts "missing ifname"; exit 1 }
+  "No such file or directory" { exit 1 }
+}  
 expect "#READY#"
 
-set timeout 30
 send "nft list set ip table-ip lan || touch /non/existent\n"
 expect {
   "lan" { puts "lan found" }
   "{ }" { puts "missing ifname"; exit 1 }
   "No such file or directory" { exit 1 }
 }  
-
 expect "#READY#"
-
-# if the test fails for no apparent reason, it is quite likely because
-# the firewall hasn't had time to reload after the new interface
-# appears and you just have to make this sleep longer. Ew, yes
-send "sleep 10; nft list set ip table-ip wan || touch /non/existent\n"
-expect {
-  "ppp0" { puts "ppp0 found " }
-  "{ }" { puts "missing ifname"; exit 1 }
-  "No such file or directory" { exit 1 }
-  timeout { exit 1 }
-}  
-expect "#READY#"
-
 exit $EXIT