liminix

commit aaa6e353db5577b6431d546272612486ee29342c
parent 69bf6cb5fbe68a2a7ed4e3eeab0596a64d25ea5e
Author: Daniel Barlow <dan@telent.net>
Date:   Sun,  8 Sep 2024 16:38:37 +0100

incz is a very rudimentary log shipper for zinc search

although it probably would work with elasticsearch as well
as zinc is alleged to be ES-compatible

this is just the package and needs hooking into the service/log
infrastructure somehow

Diffstat:
M
pkgs/default.nix
 | 
1
+
A
pkgs/incz/default.nix
 | 
38
++++++++++++++++++++++++++++++++++++++
A
pkgs/incz/incz.fnl
 | 
58
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

3 files changed, 97 insertions(+), 0 deletions(-)
diff --git a/pkgs/default.nix b/pkgs/default.nix
@@ -71,6 +71,7 @@ in {
   hi = callPackage ./hi { };
   ifwait = callPackage ./ifwait { };
   initramfs-peek = callPackage ./initramfs-peek { };
+  incz = callPackage ./incz { };
   json-to-fstree = callPackage ./json-to-fstree { };
   kernel-backport = callPackage ./kernel-backport { };
   kmodloader = callPackage ./kmodloader { };
diff --git a/pkgs/incz/default.nix b/pkgs/incz/default.nix
@@ -0,0 +1,38 @@
+{
+  fetchurl,
+  writeFennel,
+  fennel,
+  fennelrepl,
+  runCommand,
+  lua,
+  anoia,
+  lualinux,
+  stdenv
+}:
+let name = "incz";
+in stdenv.mkDerivation {
+  inherit name;
+  src = ./.;
+
+  buildInputs = [lua];
+  nativeBuildInputs = [fennelrepl];
+
+  buildPhase = ''
+    fennelrepl --test ./incz.fnl
+    cp -p ${writeFennel name {
+      packages = [
+        anoia
+        lualinux
+        fennel
+      ];
+      macros = [
+        anoia.dev
+      ];
+      mainFunction = "run";
+    } ./incz.fnl } ${name}
+  '';
+
+  installPhase = ''
+    install -D ${name} $out/bin/${name}
+  '';
+}
diff --git a/pkgs/incz/incz.fnl b/pkgs/incz/incz.fnl
@@ -0,0 +1,58 @@
+(local { : base64 } (require :anoia))
+(local ll (require :lualinux))
+
+(fn index [str indexname]
+  (string.format "{\"index\":{\"_index\":%q}}\n%s" indexname str))
+
+(local crlf "\r\n")
+
+(fn chunk [str]
+  (let [len (# str)]
+    (string.format "%x%s%s%s" len crlf str crlf)))
+
+(fn http-header [host path auth]
+  (string.format
+   "POST %s HTTP/1.1\r
+Host: %s\
+Authorization: basic %s
+Transfer-Encoding: chunked\r
+\r
+"
+   path host
+   (let [b64 (base64 :url)] (b64:encode auth))))
+
+(fn format-timestamp [timestamp]
+  ;; I can't make zincsearch understand any epoch-based timestamp
+  ;; formats, so we are formatting dates as iso-8601 and damn the leap
+  ;; seconds :-(
+  (let [secs (- (tonumber (string.sub timestamp 1 16) 16)
+                (lshift 1 62))
+        nano (tonumber (string.sub timestamp 16 24) 16)
+        ts (+ (* secs 1000) (math.floor (/ nano 1000000)))]
+    (.. (os.date "!%FT%T" secs) "." nano "Z")))
+
+(fn process-line [indexname hostname line]
+  (let [(timestamp service msg) (string.match line "@(%x+) (%g+) (.+)$")]
+    (->
+     (string.format
+      "{%q:%q,%q:%q,%q:%q,%q:%q}\n"
+      "@timestamp" (format-timestamp timestamp)
+      :service service
+      :message msg
+      :host hostname)
+     (index indexname)
+     chunk)))
+
+(fn run []
+  (let [myhostname (with-open [h (io.popen "hostname" :r)] (h:read "l"))
+        (filename loghost credentials indexname) (table.unpack arg)]
+
+    (with-open [infile (assert (io.open filename :r))]
+      (ll.write 1 (http-header loghost "/api/_bulk" credentials))
+      (while (case (infile:read "l")
+               line (ll.write 1 (process-line indexname myhostname line))))
+      (ll.write 1 (chunk "")))
+    (io.stderr:write (ll.read 0))))
+
+
+{ : run }