add health check service and example that uses it

examples/l2tp.nix

@@ -53,6 +53,7 @@ in rec {
     # ../modules/mount
     ../modules/ppp
     ../modules/round-robin
+    ../modules/health-check
     ../modules/profiles/gateway.nix
   ];
   hostname = "thing";
@@ -106,18 +107,28 @@ in rec {
               target = lns.address;
               dependencies = [services.bootstrap-dhcpc check-address];
             };
-          in svc.l2tp.build {
-            lns = lns.address;
-            ppp-options = [
-              "debug" "+ipv6" "noauth"
-              "name" rsecrets.l2tp.name
-              "password" rsecrets.l2tp.password
-            ];
-            dependencies = [config.services.lns-address route check-address];
-          };
+            l2tpd= svc.l2tp.build {
+              lns = lns.address;
+              ppp-options = [
+                "debug" "+ipv6" "noauth"
+                "name" rsecrets.l2tp.name
+                "password" rsecrets.l2tp.password
+              ];
+              dependencies = [config.services.lns-address route check-address];
+            };
+          in
+            svc.health-check.build {
+              service = l2tpd;
+              threshold = 3;
+              interval = 2;
+              healthCheck = pkgs.writeAshScript "ping-check" {} "ping 1.1.1.1";
+            };
       in svc.round-robin.build {
         name = "wan";
-        services = [ l2tp pppoe ];
+        services = [
+          pppoe
+          l2tp
+        ];
       };
       dhcp6.enable = true;
     };

modules/health-check/default.nix

@@ -0,0 +1,43 @@
+## Health check
+##
+## Runs a service and a separate periodic health process. When the
+## health check starts failing over a period of time, kill the service.
+## (Usually that means the supervisor will restart it, but you can
+## have other behaviours by e.g. combining this service with a round-robin
+## for failover)
+
+
+{ lib, pkgs, config, ...}:
+let
+  inherit (lib) mkOption types;
+  inherit (pkgs) liminix;
+#  inherit (pkgs.liminix.services) longrun;
+in {
+  options = {
+    system.service.health-check = mkOption {
+      description = "run a service while periodically checking it is healthy";
+      type = liminix.lib.types.serviceDefn;
+    };
+  };
+  config.system.service.health-check = config.system.callService ./service.nix {
+    service = mkOption {
+      type = liminix.lib.types.service;
+    };
+    interval =  mkOption {
+      description = "interval between checks, in seconds";
+      type = types.int;
+      default = 10;
+      example = 10;
+    };
+    threshold =  mkOption {
+      description = "number of consecutive failures required for the service to be kicked";
+      type = types.int;
+      example = 3;
+    };
+    healthCheck = mkOption {
+      description = "health check command or script. Expected to exit 0 if the service is healthy or any other exit status otherwise";
+      type = types.path;
+    };
+  };
+  config.programs.busybox.applets = ["expr"];
+}

modules/health-check/service.nix

@@ -0,0 +1,37 @@
+{
+  liminix, lib, lim, s6
+}:
+{ service, interval, threshold, healthCheck } :
+let
+  inherit (liminix.services) oneshot longrun;
+  inherit (builtins) toString;
+  inherit (service) name;
+  checker = let name' = "check-${name}"; in longrun {
+    name = name';
+    run = ''
+      fails=0
+      echo waiting for /run/service/${name}
+      ${s6}/bin/s6-svwait -U /run/service/${name} || exit
+      while sleep ${toString interval} ; do
+        ${healthCheck}
+        if test $? -gt 0; then
+          fails=$(expr $fails + 1)
+        else
+          fails=0
+        fi
+        echo fails $fails/${toString threshold} for ${name}
+        if test "$fails" -gt "${toString threshold}" ; then
+          echo time to die
+          ${s6}/bin/s6-svc -r /run/service/${name}
+          echo bounced
+          fails=0
+          echo waiting for /run/service/${name}
+          ${s6}/bin/s6-svwait -U /run/service/${name}
+        fi
+      done
+    '';
+  };
+in service.overrideAttrs(o: {
+  buildInputs =  (lim.orEmpty o.buildInputs) ++ [ checker ];
+  dependencies = (lim.orEmpty o.dependencies) ++ [ checker ];
+})

overlay.nix

@@ -47,6 +47,7 @@ extraPkgs // {
   # liminix library functions
   lim = {
     parseInt = s: (builtins.fromTOML "r=${s}").r;
+    orEmpty = x: if x != null then x else [];
   };
 
   # keep these alphabetical