update refs to uncaught-logs in docs/tests

instead of having to open the unix socket

bordervm-configuration.nix

@@ -102,6 +102,12 @@ in {
     systemd.services.sshd.wantedBy = pkgs.lib.mkForce [ "multi-user.target" ];
 
     virtualisation = {
+      forwardPorts = [ {
+        from = "host";
+        host.port = 7654;
+#        guest.address = "10.0.2.15";
+        guest.port =7654;
+      } ];
       qemu = {
         networkingOptions = [ ];
         options =
@@ -124,6 +130,12 @@ in {
         };
       };
     };
+
+    services.tang = {
+      enable = true;
+      ipAddressAllow = [ "10.0.0.0/24" "0.0.0.0/0" ];
+    };
+
     environment.systemPackages =
       let wireshark-nogui = pkgs.wireshark.override { withQt = false ; };
           in with pkgs; [
@@ -134,6 +146,7 @@ in {
             iptables
             usbutils
             busybox
+            clevis
           ];
     security.sudo.wheelNeedsPassword = false;
     networking = {

doc/admin.rst

@@ -97,7 +97,7 @@ time.  For example:
 Logs
 ====
 
-Logs for all services are collated into :file:`/run/uncaught-logs/current`.
+Logs for all services are collated into :file:`/run/log/current`.
 The log file is rotated when it reaches a threshold size, into another
 file in the same directory whose name contains a TAI64 timestamp.
 
@@ -108,7 +108,7 @@ human-readable format, use :command:`s6-tai64nlocal`.
 
 .. code-block:: console
 
-  # ls -l /run/uncaught-logs/
+  # ls -l /run/log/
   -rw-r--r--    1         0 lock
   -rw-r--r--    1         0 state
   -rwxr--r--    1     98059 @4000000000025cb629c311ac.s
@@ -122,10 +122,10 @@ human-readable format, use :command:`s6-tai64nlocal`.
   -rwxr--r--    1     98023 @4000000000027e6f0ed24a6c.s
   -rw-r--r--    1     42374 current
   
-  # tail -2 /run/uncaught-logs/current 
+  # tail -2 /run/log/current
   @40000000000284f130747343 wan.link.pppoe Connect: ppp0 <--> /dev/pts/0
   @40000000000284f230acc669 wan.link.pppoe sent [LCP ConfReq id=0x1 <asyncmap 0x0> <magic 0x667a9594> <pcomp> <accomp>]
-  # tail -2 /run/uncaught-logs/current  | s6-tai64nlocal 
+  # tail -2 /run/log/current  | s6-tai64nlocal
   1970-01-02 21:51:45.828598156 wan.link.pppoe sent [LCP ConfReq id=0x1 <asyncmap 0x0> <magic 0x667a9594> <pcomp> <accom
   p>]
   1970-01-02 21:51:48.832588765 wan.link.pppoe sent [LCP ConfReq id=0x1 <asyncmap 0x0> <magic 0x667a9594> <pcomp> <accom

doc/tutorial.rst

@@ -71,7 +71,7 @@ stdin/stdout.
 You should now see Linux boot messages and after a few seconds be
 presented with a root shell prompt.  You can run commands to look at
 the filesystem, see what processes are running, view log messages (in
-:file:/run/uncaught-logs.current), etc. To kill the emulator, press ^P
+:file:/run/log/current), etc. To kill the emulator, press ^P
 (Control P) then c to enter the "QEMU Monitor", then type ``quit`` at
 the ``(qemu)`` prompt.
 

modules/s6/default.nix

@@ -8,7 +8,7 @@ let
     stdenvNoCC;
   inherit (lib.lists) unique concatMap;
   inherit (pkgs.pseudofile) dir symlink;
-  inherit (pkgs.liminix.services) oneshot bundle;
+  inherit (pkgs.liminix.services) oneshot bundle longrun;
   inherit (lib) mkIf mkEnableOption mkOption types;
   cfg = config.logging;
   s6-rc-db =
@@ -237,9 +237,28 @@ in {
   };
   imports = [
     ( {config, pkgs, lib, ...}:
-      let cfg = config.logging;
+      let
+        cfg = config.logging;
+        pipeline = shipper: bundle {
+          name = "log-shipping-pipe";
+          contents = let
+            eat = longrun {
+              name = "log-shipping-pipe-eat";
+              run = ''
+                fdmove -c 12 1 \
+                ${pkgs.s6}/bin/s6-ipcserver ${cfg.shipping.socket} \
+                fdmove -c 1 12 \
+                cat
+              '';
+              producer-for = spew.name;
+            };
+            spew = shipper.override {
+              consumer-for ="log-shipping-pipe-eat";
+            };
+          in [ eat spew ];
+        };
       in mkIf cfg.shipping.enable {
-        services.${cfg.shipping.service.name} = cfg.shipping.service;
+        services.${cfg.shipping.service.name} = pipeline cfg.shipping.service;
       }
     )];
 

modules/tls-certificate/certifix-client.nix

@@ -0,0 +1,21 @@
+{ liminix, certifix-client, svc, lib, writeText, serviceFns }:
+{
+  caCertificate,
+  secret,
+  subject,
+  serviceUrl
+}:
+let
+  inherit (builtins) filter isString split;
+  inherit (liminix.services) oneshot;
+  name = "certifix-${lib.strings.sanitizeDerivationName subject}";
+  caCertFile = writeText "ca.crt" caCertificate;
+  secretFile = writeText "secret" secret;
+in oneshot {
+  inherit name;
+  up = ''
+    (in_outputs ${name}
+     SSL_CA_CERT_FILE=${caCertFile} ${certifix-client}/bin/certifix-client --subject ${subject} --secret ${secretFile} --key-out key --certificate-out cert ${serviceUrl}
+    )
+  '';
+}

modules/tls-certificate/default.nix

@@ -0,0 +1,40 @@
+
+{ lib, pkgs, config, ...}:
+let
+  inherit (lib) mkOption types;
+  inherit (pkgs) liminix;
+in
+{
+  options = {
+    system.service.tls-certificate = {
+      certifix-client =  mkOption {
+        type = liminix.lib.types.serviceDefn;
+      };
+    };
+  };
+  config.system.service.tls-certificate.certifix-client =
+    config.system.callService ./certifix-client.nix {
+      # this is probably read from files on the build machine,
+      # but are not named with ...File suffix because they are
+      # not files on the device (they get embedded into the store)
+      caCertificate = mkOption {
+        description = "CA certificate in PEM format. This must be the same CA as that which signed the certificate of the Certifix server";
+        type = types.str;
+      };
+      secret = mkOption {
+        description = "The shared secret to embed in signing request. This must match the secret configured in the Certifix service, otherwise it will refuse to sign the CSR.";
+        type = types.str;
+      };
+      subject = mkOption {
+        description = "Subject of the certificate request, as an X509 DN. The CN ('Common Name') you provide here is also used as the value of the SubjectAlternativeName extension.";
+        type = types.str;
+        example = "C=GB,ST=London,O=Liminix,OU=IT,CN=myhostname";
+      };
+      serviceUrl = mkOption {
+        description = "Certifix server endpoint URL";
+        type = types.str;
+        example = "https://certifix.lan:19613/sign";
+      };
+    };
+
+}

overlay.nix

@@ -6,19 +6,22 @@ let
     inherit (final) lib callPackage;
   };
   inherit (final) fetchpatch;
-  luaHost = prev.lua5_3.overrideAttrs(o: {
-    name = "lua-tty";
-    preBuild = ''
-      makeFlagsArray+=(PLAT="posix" SYSLIBS="-Wl,-E -ldl"  CFLAGS="-O2 -fPIC -DLUA_USE_POSIX -DLUA_USE_DLOPEN")
+  luaHost =
+    let
+      l = prev.lua5_3.overrideAttrs(o: {
+        name = "lua-tty";
+        preBuild = ''
+          makeFlagsArray+=(PLAT="posix" SYSLIBS="-Wl,-E -ldl"  CFLAGS="-O2 -fPIC -DLUA_USE_POSIX -DLUA_USE_DLOPEN")
     '';
-    # lua in nixpkgs has a postInstall stanza that assumes only
-    # one output, we need to override that if we're going to
-    # convert to multi-output
-    # outputs = ["bin" "man" "out"];
-    makeFlags =
-      builtins.filter (x: (builtins.match "(PLAT|MYLIBS).*" x) == null)
-        o.makeFlags;
-  });
+        # lua in nixpkgs has a postInstall stanza that assumes only
+        # one output, we need to override that if we're going to
+        # convert to multi-output
+        # outputs = ["bin" "man" "out"];
+        makeFlags =
+          builtins.filter (x: (builtins.match "(PLAT|MYLIBS).*" x) == null)
+            o.makeFlags;
+      });
+    in l.override { self = l; };
 
   s6 = prev.s6.overrideAttrs(o:
     let
@@ -205,6 +208,15 @@ extraPkgs // {
 
   lua = crossOnly prev.lua5_3 (_: luaHost);
 
+  luaossl' = luaHost.pkgs.luaossl.overrideAttrs(o: {
+    patches = [
+      (fetchpatch {
+        url = "https://patch-diff.githubusercontent.com/raw/wahern/luaossl/pull/218.patch";
+        hash = "sha256-2GOliY4/RUzOgx3rqee3X3szCdUVxYDut7d+XFcUTJw=";
+      })
+    ] ++ final.lib.optionals (o ? patches) o.patches;
+  });
+
   mtdutils = prev.mtdutils.overrideAttrs(o: {
     patches = (if o ? patches then o.patches else []) ++ [
       ./pkgs/mtdutils/0001-mkfs.jffs2-add-graft-option.patch

pkgs/certifix-client/certifix-client.fnl

@@ -0,0 +1,89 @@
+(local { : view } (require :fennel))
+(local { : assoc : split } (require :anoia))
+
+(local ctx (require :openssl.ssl.context))
+(local csr (require :openssl.x509.csr))
+(local altname (require :openssl.x509.altname))
+(local pkey (require :openssl.pkey))
+(local xn (require :openssl.x509.name))
+
+(local http (require :fetch))
+
+(macro ncall [f]
+  `(case ,f
+     ok# ok#
+     (nil err#) (error err#)))
+
+
+(fn x509-name [subj]
+  (let [n (xn.new)]
+    (each [_ c (ipairs (split "," subj))]
+      (let [(k v) (string.match c "(.-)=(.+)")]
+        (n:add k v)))
+    n))
+
+(fn x509-altname [subj]
+  (let [an (altname.new)]
+    (each [_ c (ipairs (split "," subj))]
+      (let [(k v) (string.match c "(.-)=(.+)")]
+        (if (= k "CN") (an:add "DNS" v))))
+    an))
+
+(fn parse-args [args]
+  (case args
+    ["--secret" secret & rest]
+    (assoc (parse-args rest)
+           :secret (with-open [f (ncall (io.open secret :r))] (f:read "l")))
+
+    ["--subject" subject & rest]
+    (assoc (parse-args rest) :subject subject)
+
+    ["--key-out" pathname & rest]
+    (assoc (parse-args rest) :key-out pathname)
+
+    ["--certificate-out" pathname & rest]
+    (assoc (parse-args rest) :certificate-out pathname)
+
+    [server] { : server }
+    _ {}))
+
+
+(local options (parse-args arg))
+
+(fn private-key []
+  (pkey.new { :type :rsa :bits 1024 }))
+
+(fn signing-request [pk]
+  (doto (csr.new)
+    (: :setVersion 3)
+    (: :setSubject (x509-name options.subject))
+    (: :setSubjectAlt (x509-altname options.subject))
+    (: :setPublicKey pk)
+    (: :addAttribute :challengePassword [options.secret])
+    (: :sign pk)))
+
+
+(fn http-post [url body]
+  (match
+      (http.request "POST" url
+                    "" 0
+                    "application/x-pem-file"
+                    body)
+    s s
+    (nil code msg) (error (.. "Error " code " POST " url ": " msg))))
+
+
+(fn run []
+  (let [pk (private-key)
+        csr (signing-request pk)
+    ;; key-out (or options.key-out-handle io.stdout)
+    ;; cert-out (or options.cert-out-handle io.stdout)
+        cert (http-post options.server (csr:toPEM))]
+    (with-open [f (ncall (io.open options.key-out :w))]
+      (f:write (pk:toPEM :private)))
+    (with-open [f (ncall (io.open options.certificate-out :w))]
+      (f:write cert))
+    (print "done")))
+
+
+{ : run }

pkgs/certifix-client/default.nix

@@ -0,0 +1,36 @@
+{
+  fetchurl,
+  writeFennel,
+  fennel,
+  fennelrepl,
+  runCommand,
+  lua,
+  anoia,
+  lualinux,
+  fetch-freebsd,
+  openssl,
+  luaossl',
+  stdenv
+}:
+let name = "certifix-client";
+in stdenv.mkDerivation {
+  inherit name;
+  src = ./.;
+
+  buildInputs = [fetch-freebsd openssl lua];
+
+  buildPhase = "";
+  installPhase = ''
+    mkdir -p $out/bin
+    cp -p ${writeFennel name {
+      packages = [
+        fetch-freebsd
+        fennel
+        anoia
+        lualinux
+        luaossl'
+      ] ;
+      mainFunction = "run";
+    } ./${name}.fnl } $out/bin/${name}
+  '';
+}

pkgs/certifix-client/main.fnl

@@ -0,0 +1,9 @@
+(local { : view } (require :fennel))
+(local ctx (require openssl.ssl.ctx))
+
+(fn run []
+  (print "hey"))
+
+
+
+{ : run }

pkgs/default.nix

@@ -60,6 +60,7 @@ in {
   # please keep the rest of this list alphabetised :-)
 
   anoia = callPackage ./anoia { };
+  certifix-client = callPackage ./certifix-client { };
   devout = callPackage ./devout { };
   fetch-freebsd = callPackage ./fetch-freebsd { };
   fennel = callPackage ./fennel { };

pkgs/liminix-tools/services/default.nix

@@ -53,6 +53,7 @@ let
     , run
     , notification-fd ? null
     , buildInputs ? []
+    , producer-for ? null
     , ...
   } @ args:
     let logger = service {
@@ -64,11 +65,11 @@ let
           pipeline-name = "${name}-pipeline";
         };
     in service (args // {
-      buildInputs = buildInputs ++ [ logger ];
+      buildInputs = buildInputs ++ lib.optional (producer-for == null) logger;
       serviceType = "longrun";
       run = serviceScript run;
       finish = cleanupScript name;
-      producer-for = "${name}-log";
+      producer-for = if producer-for != null then producer-for else "${name}-log";
     });
 
   oneshot = {
@@ -92,5 +93,6 @@ let
   });
   target = bundle;
 in {
-  inherit target bundle oneshot longrun output;
+  inherit target bundle oneshot output;
+  longrun = lib.makeOverridable longrun;
 }

tests/wlan/wait-for-wlan.expect

@@ -14,7 +14,7 @@ expect {
 }
 expect "#"
 while { $FINISHED < 10 } {
-  send "date && grep AP-ENABLED /run/uncaught-logs/* || echo \$NOT\r\n"
+  send "date && grep AP-ENABLED /run/log/* || echo \$NOT\r\n"
   
   expect {
     "wlan0: AP-ENABLED" { set FINISHED 999; set EXIT 0; }
@@ -24,7 +24,7 @@ while { $FINISHED < 10 } {
 }
 
 if { $EXIT > 0 } {
-  send "tail -40 /run/uncaught-logs/current\r\n"
+  send "tail -40 /run/log/current\r\n"
   expect "#"
 }
 exit $EXIT