commit e8fa90102f8babd2a1561617c805393412cf97cf
parent 6429e073a279535af169a0aa90b3f7b99019e7e0
Author: Daniel Barlow <dan@telent.net>
Date: Mon, 31 Aug 2026 21:19:05 +0100
add tdidf corpus searching
not hooked up to the ui yet
Diffstat:
13 files changed, 863 insertions(+), 5 deletions(-)
diff --git a/site.fnl b/site.fnl
@@ -1,14 +1,15 @@
-(import-macros { : define-tests : expect : expect= } :assert)
+(import-macros { : define-tests : expect : expect= : expect-near} :assert)
(local lfs (require :lfs))
(local { : realpath } (require :posix.stdlib))
(local { : view } (require :fennel))
(local cqueues (require :cqueues))
(local notify (require :cqueues.notify))
+(local { : stem } (require :stem))
(local textile (require :textile))
(local lcmark (require :lcmark))
(local { : to-html } (require :html))
-(local { : prepend : parse-timestamp : trace : %% } (require :lib))
+(local { : clone : prepend : parse-timestamp : trace : %% } (require :lib))
(fn markup-bare-urls [str]
@@ -128,6 +129,30 @@
(string.gsub "_+$" "")
string.lower))
+(fn count-terms [s]
+ (let [terms {}]
+ (each [w (string.gmatch s "%w+")]
+ (let [lowercase (stem (w:lower))]
+ (tset terms lowercase (+ (or (. terms lowercase) 0) 1))))
+ (let [most-common-count
+ (accumulate [max 0
+ term count (pairs terms)]
+ (if (> count max) count max))]
+ (each [term count (pairs terms)]
+ (tset terms term (/ count most-common-count))))
+ terms))
+
+(define-tests :terms
+ (expect= (count-terms "apple banana. sofa")
+ {:appl 1.0 :banana 1.0 :sofa 1.0})
+ (expect= (count-terms "apple sofa Apple")
+ {:appl 1.0 :sofa 0.5})
+ (expect= (count-terms "sit sitting")
+ {:sit 1.0})
+ )
+
+(fn read-stemmed-terms [stream] (count-terms (stream:read "*a")))
+
(fn index-posts [path]
(collect [name (lfs.dir path)]
(let [fullname (.. path "/" name)
@@ -157,8 +182,30 @@
(tset :topics topics)
(tset :body post-body)
(tset :body-offset (f:seek :cur 0))
+ (tset :terms (or (read-stemmed-terms f)
+ (error (.. "no terms in " name))))
(tset :date (and fields.date (parse-timestamp fields.date)))))))))))
+;; for each term in each document,
+;; idf is log(number of docs with the term/number of docs)
+
+(fn doc-frequencies [posts]
+ (let [df {}]
+ (each [_ post (pairs posts)]
+ (each [term _ (pairs post.terms)]
+ (tset df term (+ 1 (or (. df term) 0)))))
+ df))
+
+(define-tests :doc-frequencies
+ (let [posts (index-posts "tests/posts/")
+ freqs (doc-frequencies posts)]
+ (expect= freqs.year 1)
+ (expect= (. freqs (stem "windmill")) 1)
+ (expect= (. freqs (stem "porsche")) 1)
+ (expect= (. freqs (stem "bike")) 2)
+ (expect= (. freqs (stem "duck")) 3)))
+
+
(fn static-path-if-safe [basedir name]
(let [prefix (realpath basedir)
@@ -182,8 +229,60 @@
[count topic])
(table.sort (fn [a b] (> (. a 1) (. b 1)))))))
-(fn search-posts [posts { : filter : order : offset : limit &as opts }]
+(fn dot-product [a b]
+ (accumulate [sum 0
+ term fa (pairs a)]
+ (let [fb (. b term)]
+ (if fb
+ (+ sum (* fa fb))
+ sum))))
+
+(fn magnitude [v]
+ (math.sqrt
+ (accumulate [product 0
+ _ l (pairs v)]
+ (+ product (* l l)))))
+
+(fn similarity [a b]
+ (/ (dot-product a b) (* (magnitude a) (magnitude b))))
+
+(define-tests :similarity
+ (expect-near (magnitude {:a 1 :b 1}) (math.sqrt 2))
+ (expect= (similarity {:a 1} {:b 1}) 0.0)
+ (expect= (similarity {:a 1} {:a 1}) 1.0)
+ ;; haven't verified these two
+ (expect-near (similarity {:a 1 :b 0.5} {:a 2 :b 6 :c 2}) 0.67419986246324)
+ (expect-near (similarity {:a 3 :p 2} {:p 0.5}) 0.55470019622523)
+ )
+
+
+(fn filter-terms [posts df terms]
+ (let [terms (collect [term (string.gmatch terms "(%w+)")]
+ (values (stem term) 1))
+ docs (accumulate [sum 0 _ _ (pairs posts)] (+ sum 1))]
+
+ (icollect [_ post (pairs posts)]
+ (let [tfidf (collect [term tf (pairs post.terms)]
+ (values term (* tf (math.log (/ docs (. df term))))))
+ sim (similarity terms tfidf)]
+ (if (> sim 0)
+ (doto (clone post) (tset :similarity sim)))))))
+
+(define-tests :filter-terms
+ (let [posts (index-posts "tests/posts")
+ df (doc-frequencies posts)
+ matches (filter-terms posts df "sbcl")]
+ (expect= (# matches) 2)
+ (let [names (icollect [_ p (pairs matches)] p.path)]
+ (table.sort names)
+ (expect=
+ names ["20260102_Magic_DNS.md" "e0b74727-b5b3-4091-92f3-936a758b286b.md"]
+ ))))
+
+
+(fn search-posts [posts df { : filter : order : offset : limit &as opts }]
(let [topics (topics posts)
+ posts (if (and filter filter.terms) (filter-terms posts df filter.terms) posts)
posts
(icollect [_ post (pairs posts)]
(when
@@ -232,16 +331,20 @@
(fn for-pathname [path]
(var posts {})
+ (var frequencies {})
(doto
{
:get #(. posts $2)
- :search #(search-posts posts $2)
+ :search #(search-posts posts frequencies $2)
:path path
:static-file (fn [_ name]
(static-path-if-safe path name))
:topics #(topics posts)
:weighted-topics #(weighted-topics posts)
- :refresh (fn [self] (set posts (index-posts path)) self)
+ :refresh (fn [self]
+ (set posts (index-posts path))
+ (set frequencies (doc-frequencies posts))
+ self)
:watch watch-changes
}
(: :refresh)))
diff --git a/tests/posts/116652ff-a046-4bbb-a4ae-c6becca6ce43.md b/tests/posts/116652ff-a046-4bbb-a4ae-c6becca6ce43.md
@@ -0,0 +1,16 @@
+Topics: motorbike ride-report
+Date: 2025-05-05T19:16:26.903183Z
+Title: U is for Underground
+
+While looking back through my photos I realised I hadn't said anything about "U is for Underground" which was last Saturdays ride because this Saturday was all about brake lines.
+
+
+
+
+
+
+
+
+This is Scotts Grotto near Hertford. Its a collection of small chambers with shell-lined walls connected by tunnels, which was built in the 1760s by John Scott, a Quaker poet and landowner. It's very dark.
+
+Twas a fun route both there and back as I had the time not to have to rush back down the A10 and was able to appreciate the handling improvements from having had the suspension setup. On the way back I followed a Porsche Targa through the twisty bit, but the driver was doing 50 in the NSL (60 mph) sections and 40 in the 50 limits. From which we can clearly conclude that a Honda CBR600f is better than a Porsche (/s)
diff --git a/tests/posts/20210212_Her_name_is_Oreo.md b/tests/posts/20210212_Her_name_is_Oreo.md
@@ -0,0 +1,91 @@
+Topics: android
+Title: Her name is Oreo
+Date: Fri Feb 12 13:47:14 2021
+
+I am at a point somewhere along the way of a long yak-shaving journey
+("I wonder if the MSM8953 SoC supports ANT+ in hardare and it just
+needs enabling in the Android software?") which involves rebuilding
+Android for my phone. There seem to be very few ROMs for the Moto G5
+Plus ("potter") that are still being maintained, and even fewer of
+them that have build instructions that (a)
+[work as described](https://github.com/crdroidandroid/android/issues/57#issuecomment-773367834ArrowOS);
+(b) comprise
+[fewer than 30 manual steps](https://blog.arrowos.net/compilation-guide/). Do Android ROM
+developers not do CI builds? If this was all GPL I could make a strong
+case it's in breach. ("The source code for a work means the preferred
+form of the work for making modifications to it [...] all the source
+code for all modules it contains, plus any associated interface
+definition files, plus the scripts used to control compilation and
+installation of the executable."). Yes, I know, volunteers, spare
+time, teeth of gift horses, etc etc. End rant
+
+Anyway, so far I am doing the following and it has not yet blown up
+
+1) using the [docker image](https://github.com/lineageos4microg/docker-lineage-cicd) from [LineageOS for Microg](https://lineage.microg.org/)
+
+2) invoking it as follows
+
+```
+docker run \
+ -e "BRANCH_NAME=lineage-15.1" \
+ -e "DEVICE_LIST=potter" \
+ -e "SIGN_BUILDS=false" \
+ -e "SIGNATURE_SPOOFING=restricted" \
+ -e "INCLUDE_PROPRIETARY=false" \
+ -e "WITH_SU=true" \
+ -e "CUSTOM_PACKAGES=GmsCore GsfProxy FakeStore MozillaNlpBackend NominatimNlpBackend com.google.android.maps.jar FDroid FDroidPrivilegedExtension " \
+ -v "/$HOME/src/android/l4mg/lineage:/srv/src" \
+ -v "/$HOME/src/android/l4mg/zips:/srv/zips" \
+ -v "/$HOME/src/android/l4mg/logs:/srv/logs" \
+ -v "/$HOME/src/android/l4mg/cache:/srv/ccache" \
+ -v "/$HOME/src/android/l4mg/keys:/srv/keys" \
+ -v "/$HOME/src/android/l4mg/manifests:/srv/local_manifests" \
+ lineageos4microg/docker-lineage-cicd
+```
+
+3) with this manifest in `$HOME/src/android/l4mg/manifests/roomservice.xml`
+
+```
+<?xml version="1.0" encoding="UTF-8"?>
+<manifest>
+ <project name="LineageOS/android_external_bson" path="external/bson" remote="github" revision="cm-14.1" />
+ <project name="LineageOS/android_device_qcom_common" path="device/qcom/common" remote="github" revision="lineage-15.1" />
+ <project name="LineageOS/android_packages_resources_devicesettings" path="packages/resources/devicesettings" remote="github" revision="staging/lineage-15.1" />
+ <project name="boulzordev/android_device_motorola_potter" path="device/motorola/potter" remote="github" revision="lineage-15.1-64" />
+ <project name="boulzordev/android_vendor_motorola_potter" path="vendor/motorola/potter" remote="github" revision="lineage-15.1-64" />
+ <project name="boulzordev/android_kernel_motorola_msm8953" path="kernel/motorola/msm8953" remote="github" revision="lineage-15.1-64" />
+</manifest>
+```
+
+Worthy of note:
+
+- I found the manifest on [the Interwebz](https://pastebin.com/7xV9G1A4) by use of a search engine. I had to edit the revision attribute of `android_device_qcom_common` from `staging/lineage-15.1` to `lineage-15.1`
+
+- because I have a (presumed working) example manifest for 15.1 I am
+ not attempting to build any newer version. One change at a time,
+ hey
+
+- in my limited understanding, the manifest tells the build system
+ where to find Moto proprietary files, and the
+ `INCLUDE_PROPRIETARY=false` build argument tells the L4microg build
+ scripting not to try and download them from "TheMuppets". This is
+ important because the muppets don't have them for my device.
+
+- getting F-droid and Microg baked in is probably no bad thing
+
+As I write this, it has been building for ~ 30 minutes and not fallen
+over yet, so things are looking vaguely promising
+
+```
+[ 8% 8343/95970] target thumb C++: libv8src_32 <= external/v8/src/builtins/builtins-constructor.cc
+```
+
+Apparently it's done 8% of something
+
+## Update: Fri Feb 12 19:57:48 GMT 2021
+
+It works! It built a buch of stuff in `zips/potter/` including the file
+`lineage-15.1-20210212-UNOFFICIAL-potter.zip` which I installed from Recovery
+using the instructions at [https://wiki.lineageos.org/devices/peregrine/install](https://wiki.lineageos.org/devices/peregrine/install)
+Yes, that's not the same phone, but it's close enough to use the same procedure
+
diff --git a/tests/posts/20211031_Cross_product.md b/tests/posts/20211031_Cross_product.md
@@ -0,0 +1,122 @@
+Topics: nix
+Title: Cross product
+Date: Sun Oct 31 15:47:10 2021
+
+I had cause this afternoon to remember the
+[Monad Tutorial Fallacy](https://byorgey.wordpress.com/2009/01/12/abstraction-intuition-and-the-monad-tutorial-fallacy/),
+which has been summarised as saying that when you finally understand
+them, you lose the ability to explain it to others.
+
+I hypothesise that the same is probably true of cross-compilation in
+the Nix package system, and therefore I present these notes not as a
+superior alternative to any of the existing documentation, but because
+I wrote them down to aid my understanding and now need to put them
+somewhere I can refer back to them.
+
+So. Let's suppose we're building NixWRT. Most likely we're building on
+an x86-64 system, to produce an image which will run on a MIPS
+device. In the event that there are any programs in that image which
+generate code (which is unlikely as we're not shipping compilers), we
+want them also to generate MIPS code. Thus, in the standard
+terminology we have:
+
+* build: x86-64
+* host: MIPS
+* target: MIPS
+
+(This naming convention comes from Autoconf, and so we are stuck with
+it. To make it make sense, consider the built product rather than the
+build process: we are describing a thing that was built on x86-64, is
+hosted on MIPS, and would - if it emitted any code - emit code that
+runs on MIPS)
+
+However, not all of the software we create (or depend on) will be
+needed on the MIPS system - some of it (e.g. buid tools, compilers and
+other kinds of translators) will have to run on x86-64. So how do we
+keep track of it all?
+
+Let's look at some examples:
+
+* Package A contains source code which is translated into some other
+ form using programs provided by package B (e.g. B provides an SVG to PNG
+ convertor). The programs in B must run on the build machine:
+ thus, the `host` for B is the `build` for A. Provided that B is not
+ generating executable code - i.e. we don't have to worry about the
+ target - then we represent this by including B in the
+ `nativeBuildInputs` attrribute of A's derivation.
+
+* Package A contains source code which is compiled using programs
+ provided by package B for execution on the build system. For
+ example, B is a C compiler which we are using to build `nconf`,
+ which we will then run to create the `.config` file that the linux
+ kernel build process uses. In this case the program in B must run
+ on the build system of A and also must _target_ the build system of
+ A. We represent this by putting B in the `depsBuildBuild` attribute
+ of A's derivation.
+
+* if we have a package A which depends at runtime on another package B
+ (e.g. the build for A creates a shell script, one of whose commands
+ is provided by B) both those derivations have the same `host`. In
+ this case B doesn't care about the target, so we add it to the
+ `buildInputs` for A (if it does care, that's more complicated). As
+ the developers of A we must ensure that programs in B are reachable
+ from A, either by embeddding the full pathname of B into the script
+ or using a wrapper that sets $PATH.
+
+* if A required at run-time some source code contained in B (e.g. A is
+ a script for some interpreter, and B is a source-distributed
+ library, for it) then B has no `host` to speak of. If there is any
+ native code component in that library, though, it must be code that
+ runs on the same system as A's host - so `buildInputs` again. See
+ [abcde](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/audio/abcde/default.nix)
+ for an example. Note also the `wrapProgram` call which sets
+ `PERL5LIB` to ensure that the code in A can find the code in B at
+ runtime.
+
+* if A depends when it is *built* on source code contained in B
+ (suppose: the build invokes a Ruby script, and B is a gem required
+ by that script) then B must be runnable on the build system of A.
+ Host(B) = Build(A) implies `nativeBuildInputs` unless there is some
+ target shenanigans. Consulting the
+ [manual](https://nixos.org/manual/nixpkgs/unstable/#setup-hook-perl)
+ it seems that for some interpreters there is support for adding
+ the files in B to interpreter's search path while A is built.
+
+* if A is a program that runs on the host, and is linked to binary
+ static libraries provided by B, the `host` for B must be the same as
+ for A, so my reading is that this is `buildInputs`. Note that A must
+ be able to find B at compile time, which is handled by the
+ [CC Wrapper](https://nixos.org/manual/nixpkgs/unstable/#cc-wrapper)
+ adding appropriate flags.
+
+* if A is a program that runs on the host, and depends on binary
+ _shared_ libraries provided by B, the `host` for B must be the same
+ as for A so this is similar to the previous case. The absolute
+ pathname of the shared library provided by B will be embedded into
+ the binary of A.
+
+Why am I caring about this right now? I rearranged bits of NixWRT and
+updated it to a recent Nixpkgs revision, causing OCaml to stop
+building for [reasons I didn't fully understand](https://github.com/NixOS/nixpkgs/issues/143883)
+
+So, here is what I think is happening:
+
+- the kernel is being built on x86-64 (build) for execution on MIPS
+ (host). It doesn't generate code when it's run (er, as far I know -
+ at any rate it can't be configured to generate code for some other
+ system than the one its running on) so I don't care about target.
+
+- to create the kernel source tree we run Coccinelle on x86-64 - so
+ Coccinelle's `host` is the kernel's `build`. Coccinelle produces C
+ source files, so again I don't care about `target`. This means we
+ should use `nativeBuildInputs` to declare it as a dependency.
+
+- Coccinelle is written in OCaml, which is a compiler and generates
+ code for some target system according to its build options. This
+ means that OCamls `host` _and_ `target` are both coccinelle's `build`
+ system, so we use `depsBuildBuild` to declare it as dependency.
+
+
+
+Clear? If this doesn't help, I invite you to consider the possibility
+that cross-compilation is like a burrito.
diff --git a/tests/posts/20230215_Sub-liminix_messaging.md b/tests/posts/20230215_Sub-liminix_messaging.md
@@ -0,0 +1,17 @@
+Topics: nix nixwrt liminix
+Title: Sub-liminix messaging
+Date: Wed Feb 15 22:23:48 2023
+
+> I am restarting/rewriting NixWRT,
+
+he said, a few months ago. This is a short follow-up announcement
+to say that
+
+* I registered a domain name: you can now find regular Liminix updates
+ at [www.liminix.org](https://www.liminix.org)
+* there will _be_ regular Liminix updates, because Liminix is now
+ [funded by NLnet](https://nlnet.nl/project/Liminix/) through
+ the [NGI0 Entrust Fund](https://nlnet.nl/entrust/)
+
+I am very stoked about this. I'm aiming for ~ weekly updates in that
+place.
diff --git a/tests/posts/20260102_Magic_DNS.md b/tests/posts/20260102_Magic_DNS.md
@@ -0,0 +1,176 @@
+Topics: rust esp32 eculocate
+Subject: Magic DNS
+Date: Fri Jan 2 10:58:45 2026
+
+It's all very 2026 around here, isn't it? I am reminded by
+
+* [The Case for Blogging in the Ruins](https://www.joanwestenberg.com/the-case-for-blogging-in-the-ruins/) and
+* [A website to destroy all websites](https://henry.codes/writing/a-website-to-destroy-all-websites/)
+
+to jot down some of what I've been doing in the past month or so. The
+tl;dr is "making a thing I can plug into my motorbike ECU to log the
+data (rpm, speed, throttle position, temperatures etc etc) it
+produces". For reasons mostly of ramifying the learning opportunities,
+I decided the best way would be to get a cheap ESP-32 device (it's
+RISC-V - isn't that cool?) and hook it up to a level converter, and
+then write a program for it in Rust (Rust learning opportunity ahoy)
+to twiddle the serial line appropriately and send the data over the
+network to the mobile phone which sits on my handlebars.
+
+It turns out that I spent _way_ less time getting the serial interface
+to the [Honda K-line ECU signal](https://gonzos.net/projects/ctx-obd/)
+to reveal its secrets than on the "why don't you just ..." part where
+I want to stream the data over wifi to another device. So this post is
+actually not at all about hardware hacking.
+
+The constraints I have imposed on myself here are
+
+* I do not want to hardcode my wifi ssid and password into the device
+(actually, I have at least two wifi networks I may want to use it with)
+
+* the (yet to be written) data collection app should be able to find
+the device without hardcoding its IP address or requiring me to type it
+in - as the device is getting its address from DHCP, we don't even
+know what address it will get
+
+These are both _in principle_ solved problems.
+
+There's a convention for [provisioning wifi on these devices](https://randomnerdtutorials.com/esp32-wi-fi-provisioning-ble-arduino/)
+which involves using a mobile phone app to connect to it using BLE
+then sending the ssid/password of the chosen access point. In fact
+there's even a prebuilt Android app which we can use and an esp32
+arduino library which we can't (because we have elected to make our
+lives difficult and use Rust instead). But I am led to believe that
+"rewrite everything in Rust" is idiomatic for Rust programmers anyway.
+I haven't done this yet.
+
+And for the "what's my IP address" problem there is a standard way, by
+combining [Multicast DNS](https://datatracker.ietf.org/doc/html/rfc6762)
+and [DNS-based Service Discovery](https://datatracker.ietf.org/doc/html/rfc6763), for
+computers to publish their services on the LAN. When I say
+"computers": if this household is typical, mostly they're set-top
+boxes, printers, light bulbs, smart speakers and thermostats rather
+than general-purpose computing devices. I've mostly done this bit.
+
+## Terms
+
+**Multicast DNS** is DNS, but peer-to-peer: it reuses mostly the same packet formats
+but instead of requiring a centralised server which knows all the
+names, every device listens on a multicast address
+for DNS queries for its _own_ name.
+
+**DNS-SD** is a convention for which records you can query/need to send in
+order to advertise what kind of services you have and where they are.
+Because sending an A record alone is not sufficient for anyone with a
+Mac and a fancy-schmancy service browser to know what _kind_ of service
+is on offer at that address. Is it a printer? A dishwasher? An
+IoT air fryer?
+
+The RFCs for each (which are, by the way, much easier reads than a lot
+of RFCs and contain no EBNF at all) go to great lengths to point out
+that each is independent of the other. But they stack well.
+
+## DNS-SD, 3048 metre view
+
+DNS-SD is based on a paradigm of "services" and "service instances". A
+"service" is the general "kind" of thing on offer and is named
+something like `_http._tcp.local` - it will always end in `_tcp.local`
+if it is TCP or `_udp.local` if it is anything other than TCP. For our
+ECU project we chose the service name `_keihin._udp.local` after the
+manufacturer of the ECUs that the device knows how to talk to. A
+service instance might be something like
+`WiserHeat05AB12._http._tcp.local`. Service names aren't usually
+hierarchical but there are a few with a second level like
+`_printer._sub._http._tcp`
+
+
+The minimum/usual set of records you need to publish for DNS-SD is
+this (pseudocode)
+
+```
+myinstancename._theservicename._udp.local SRV, data: (target: myhostname.local, port: nnnn)
+myhostname.local A, data: a.b.c.d
+myinstancename._theservicename._udp.local TXT, data: "txtvers=1"
+_theservicename._udp.local PTR, data: myinstancename._theservicename._udp.local
+_services._udp.local PTR, data: _theservicename._udp.local
+```
+
+Your service instance needs a SRV and a TXT, then there's a PTR
+connecting the service instance to the service for people who are
+browsing the service - think about e.g. an "Add a printer" dialog box,
+then there's a PTR from `_services._udp.local` to the service name PTR
+for people who are running `avahi-browse -a` or its moral equivalent
+in GUI-land. And not forgetting there's an A record matching the one in the
+SRV record data.
+
+* Note: `_services._udp.local` is the right name for discovery
+even if your service is TCP - there is no `_services._tcp.local`
+
+* Note: I am assuming `.local` is the suffix, which is likely true for
+MDNS but probably not if you are using DNS-SD with regular DNS
+
+## MDNS
+
+The single biggest problem when implementing MDNS is the lack of
+tooling to test it against. In my experience:
+
+* dig: historically, you used to be able to `dig @224.0.0.251 -p 5353 name.local` but it largely worked by accident and [now it
+doesn't](https://gitlab.isc.org/isc-projects/bind9/-/issues/3428). Note
+that even when it _did_ work it wasn't sending the same packets as a
+real MDNS query.
+
+* avahi-browse: e.g. `avahi-browse -v -a` shows all the services on
+ the LAN. Note that is is a frontend to the persistent `avahi-daemon`
+ and there is caching happening in there somewhere, so if it didn't
+ work then and you made a change and restarted your service ... is
+ your service still broken or did it not reissue the query? shrug-emoji.gif
+
+* `mquery` from Jeremie Miller's mdnsd: note that this has been forked
+ into a million pieces. The one I'm using is
+ [https://github.com/troglobit/mdnsd](https://github.com/troglobit/mdnsd).
+ This works for querying the `_services._udp.local` discovery name
+ but if you run e.g. `mquery _scanner._tcp.local` it appears to send
+ the query and then sit there silently ignoring the responses. *But*:
+ it doesn't cache as avahi-browse does, so that's good.
+
+* wireshark, of course. Wireshark is pretty good, but note that it
+ will display your replies as "Unsolicited" because the query id for
+ MDNS is (per the standard) 0, so there is no way for it to correlate
+ them with requests.
+
+* [mdns-debugger](https://github.com/andrewbonney/mdns-debugger) was
+ handy for pointing out my TTLs (and a lot of other TTLs) were wrong.
+ It didn't point out that my PTR record data was incorrectly encoded
+ and was therefore naming a nonexistent A record, which was a source
+ of much hair tearing.
+
+* there are a couple of Android apps I also used, mostly to see what
+ they'd do when nothing was working (see "hair tearing" above) and I
+ wa sout of ideas. "mDNS Discovery" (`com.mdns_discovery.app`) and
+ "Service Browser" (com.druk.servicebrowser). The first one is
+ prettier, the second one _very_ helpfully rendered the errant PTR as
+ with a backslash as `eculogical\.local` and so led me to the said
+ encoding error.
+
+
+## Where are we now?
+
+I believe that it now does everything an mdns responder SHOULD(sic) do
+except
+
+* compress labels in response data
+* ignore queries for records where the record we'd send is already in the answers section of the query message
+* NSEC
+
+and I can't decide, in the context of this being a program that
+probably nobody else in the world will ever use and even I will only
+use on one single piece of hardware (I only have one motorbike)
+whether implementing those things is a good and laudable decision
+because spec compliance is important, or just a way of further putting
+off the inevitable next step which involves writing the Android app
+to collect the data.
+
+It also could do with being extracted into its own module/crate/thing
+to be more modular. I'd say "to aid reuse" but I don't think anyone
+really wants to (or should want to) reuse my [novice-level Rust code](https://gti.telent.net/dan/eculogical/src/branch/main/ecu-esp32/src/wifi.rs).
+Learning in public.
diff --git a/tests/posts/20260412_No_harm_no_fowl.md b/tests/posts/20260412_No_harm_no_fowl.md
@@ -0,0 +1,45 @@
+Topics: motorbike ride-report pond
+Subject: No harm no fowl
+Date: Sun Apr 12 16:33:05 2026
+
+
+
+My quest to visit all the duck ponds in Essex and Hertforshire has
+involved a lot of looking at Google Streetview, and when I saw [this
+one near Epping Green](https://maps.app.goo.gl/WsvBYd1mhmxKZXqY8) it
+looked a bit overgrown and unloved but I figured I'd stop by anyway on
+the way to pick up new brake pads in Harlow.
+
+
+
+
+
+Just like the previous pond, there were no ducks - I may have to
+broaden my selection criteria - but despite that, I am feeling
+tremendously pleased about having stopped there anyway. It's
+obviously seen a bit of maintenance since the Google car went past,
+and was so much lovelier to visit in person than I was expecting. It's
+sited in a small grassy area with some trees around and it's just
+... sort of heartwarming. Probably helped that the sun was out.
+
+
+
+The pub is across the road and has been closed since 2019, which is a
+bit of a shame as it would be a lovely place to sit outside with a
+drink and look across the road. As far as I can tell from the
+internet there are/were plans to reopen it as a [community pub](https://www.yourharlow.com/2024/12/07/travellers-friend-pub-saved-due-to-permission-for-house-building-in-car-park/) but I found no news on that more recent than Dec 2024.
+
+From Epping Green to Harlow via a tedious litany of suburban
+roundabouts, which would have been less faff if my satnav wasn't
+giving me the silent treatment. Collect brake pads from Sportbikeshop,
+then a mildly circuitous and much more rural route home via Old
+Harlow, Matching Tye, Matching Green, Moreton, North Weald Bassett,
+Epping etc. There's a very nice stretch of road between Harlow and
+Matching: bendy enough to be fun but open enough that you can see
+further than one bend ahead.
+
+I have booked an
+[ERS](https://www.artridertraining.co.uk/advanced-training/#1) which
+is happening in about ten days and I hope will be
+fun/interesting/informative. I'm not going to be at all defensive
+about having my riding criticised, no, definitely not.
diff --git a/tests/posts/32c9fad8-6968-49ed-ab1b-8eaa77d7a8a5.md b/tests/posts/32c9fad8-6968-49ed-ab1b-8eaa77d7a8a5.md
@@ -0,0 +1,38 @@
+Topics: motorbike ride-report
+Date: 2024-06-08T16:30:31.988179Z
+Title: D is for Duck pond
+
+
+Did I mention there was going to be a duckpond?
+
+
+It took longer to get to Writtle than I'd expected, mostly due to the need to pull over several times and fiddle with the phone screen ( I'm sure the ultimateaddons case is very waterproof but its also impossible to see in sunlight and not helped by low-contrast colors used by map apps) so I didn't stay for longer than needed to get photos. Very pleasant village green when eventually I found it
+
+
+
+
+Improving:
+
+* much smoother on the clutch/matching revs: still have occasional leaps from first to second, but they're less common than they were
+* cornering confidence (yes, I know about countersteering :-)
+* much more adept at getting the bike up the kerb and wheeling it back into the back garden at the end of the ride
+
+Need to improve:
+
+* low-speed manoeuvres - couple of places I could probably have U-turned in the road, I instead got off and pushed it around.
+
+Special moment:
+
+* attempted an overtake of some car driver doing 35mph on a straightish A road with a 60 limit, had to pull back in behind them when they sped up to 55 and oncoming traffic appeared. They did go a bit quicker after that though
+
+
+
+Navigational:
+
+* managed to avoid the M11 this time, by dint of [getting lost in North Weald Bassett and ending up on Epping Road](https://www.openstreetmap.org/directions?engine=fossgis_osrm_bike&route=51.7296%2C0.1601%3B51.6152%2C0.0246#map=15/51.7078/0.1418&layers=N). Rode home through Epping Forest, which is almost as scenic but not quite as much fun as cycling through it (because when cycling you can ride through the actual forest not just along the roads that bisect it)
+
+
+
+Just to prove that I actually _went_ to the village green and duck pond in Writtle and didn't steal the photos off the internet, here a picture of my bike in front of the green. If you zoom in far enough you run out of pixels, but if you didn't run out of pixels you'd see the pond at the far end.
+
+According to my list, E is for earthworks, which sounds a bit dull. Maybe I can think of something better
diff --git a/tests/posts/58b992be-c0da-42ab-8bea-65c3cc1f26db.md b/tests/posts/58b992be-c0da-42ab-8bea-65c3cc1f26db.md
@@ -0,0 +1,30 @@
+Topics: motorbike ride-report
+Date: 2024-05-09T17:16:47.913222Z
+Title: First ride
+
+Yesterday I went out for my first ride not counting riding home from
+the dealership.
+
+tl;dr didn't drop it didn't crash, so it met the basic success criteria. But not an unalloyed pleasure.
+
+For the longer-form report ... the plan was to go to a [garden centre](http://www.blueroomscafe.co.uk/) about 20 miles from home and
+buy a coffee to sit in the sunshine with. I perhaps foolishly decided
+to try navigation using
+[Kurviger](https://kurviger.de/about/en). Foolish (1) because
+Kurviger's idea of a "curvy" route in London turns out to be a route
+that eschews all major thoroughfares in favour of 20mph residential
+roads, and foolish (2) because Kurviger "Follow mode" is not terribly
+(at all) clear on a smartphone screen when the sun's shining directly
+on it.
+
+I spent 20-30 minutes riding around Chingford and South Woodford
+before deciding I was lost enough to stop and look at the map
+properly, and when I did, I appeared to be on a long straight road
+that would have passed under the A406 (London's "North Circular Road")
+and rejoined the route. So I set off again Unfortunately, when I
+reached the A406 it turned out to be not a bridge but an
+on-ramp. Downside: I was now pointing in the wrong direction; upside:
+I knew exactly where I was and it was about 5 minutes from home. So I
+went home and had a coffee in the sunshine there instead.
+
+Lessons learned: get some miles from home before engaging wiggle mode.
diff --git a/tests/posts/af8a69c7-94b3-485c-a455-f3175a471bc3.md b/tests/posts/af8a69c7-94b3-485c-a455-f3175a471bc3.md
@@ -0,0 +1,13 @@
+Topics: motorbike ride-report airbag
+Date: 2025-05-17T14:42:12.358784Z
+Title: W is for Windmill
+
+
+W is for Windmill. A traditional windmill made of wood and painted a kind of creamy colour. It has four wooden vanes, attached to the central point with what looks like a mechanism to change the angle of the vane and so speed up/slow down rotation. It was stopped when I saw it.
+
+It's in Mountnessing, in a park and across the road from a pub which
+served me a perfectly-Coke-flavoured glass of Coke.
+
+Dropped the bike on the way home :-(. It was a basic mistake (put my foot down, the ground was a bit further away than I thought) but it did kind of put a damper on the rest of the journey home. No damage to the bike (thanks crash bungs!) but my airbag did go off as I inelegantly dismounted to the side. Notes to self: find out how to replace the airbag cartridge.
+
+My lower back feels a bit funny, I think from where I was trying to save the bike from falling, but yoga will fix that
diff --git a/tests/posts/e0b74727-b5b3-4091-92f3-936a758b286b.md b/tests/posts/e0b74727-b5b3-4091-92f3-936a758b286b.md
@@ -0,0 +1,22 @@
+Topics: motorbike ride-report
+Date: 2025-01-18T18:42:48.568052Z
+Title: O is for Octagonal council house
+
+I have an hours chinmount GoPro footage of today's ride and to a first approximation it's all really really dull to watch. It has demonstrated though just how much time I spend looking down at my phone/gps while riding, which is a bit scary. Kurviger seems to have stopped doing voice directions into my helmet and I can't see why (all the options that say it should do are enabled, and the headset works fine for music so it's not a hardware or a bluetooth issue). Looking down at the map is ... a poor substitute
+
+I might have another look through it later and see if there's anything worth keeping
+
+
+O is for Octagonal. This is the "Dutch House" in Rayleigh, said to be the oldest council house in England (although it's not been in council ownership all that time). I had to park a few minutes walk away and come back to it, so I didn't spend long here. Plus it was starting to get dark by the time I arrived.
+
+
+it took longer to get there than Kurviger said it would, but I feel I have a reasonable excuse for at least one of the delays. I was far too risk-averse (cowardly) to go through this when I didn't know how deep it was, it seemed like a bad place to potentially break down
+
+
+
+I went a bit closer on foot, and then I said a rude word and pushed my motorbike round through 180° and went back the way I came
+
+Once home, I asked about the voice prompt on the Kurviger forum and then
+found the problem myself. Classic example of rubber duck debugging.
+
+https://forum.kurviger.com/t/voice-prompts-have-stopped-working-3-5-4/20222
diff --git a/tests/posts/fennel.md b/tests/posts/fennel.md
@@ -0,0 +1,11 @@
+title: Fennel
+slug: fennel
+date: Mon Jul 28 22:32:16 2025
+
+[Fennel](https://fennel-lang.org/) is "a programming language that
+brings together the simplicity, speed, and reach of Lua with the
+flexibility of a lisp syntax and macro system". It's presently (where
+"presently" is whatever it says in the timestamp of this post) my
+"goto" programming language.
+
+lobste.rs has an [interesting interview with its maintainer](https://lobste.rs/s/terwiu/lobsters_interview_with_technomancy)
diff --git a/tests/posts/welcome-back-victoria.md b/tests/posts/welcome-back-victoria.md
@@ -0,0 +1,174 @@
+Date: Tue Oct 21 18:52:38 2025
+Title: Centralised logging with Liminix and VictoriaLogs
+Topics: liminix
+
+It's a year since I wrote [Log off](https://www.liminix.org/2024/10/06/log-off.html), in which I
+described some ongoing-at-the-time work to make Liminix devices log over the
+network to a centralised log repo. It's also, and this is entirely a coincidence,
+a year since I made any kind of progress on it: since that time all my log messages have continued to be written to ramdisk that will
+be lost forever like tears in the rain.
+
+This situation was not ideal. I had some time and energy recently to
+see if I could finish it up and, well, I haven't done that exactly but
+whereas last time I only *believed* it was substantially finished,
+this time I believe it is substantially finished.
+
+It goes a little something like this:
+
+## Tap the log pipeline
+
+Each service in Liminix is connected to its own log process, which is
+(for 98% of the services) connected to the "fallback logger" which
+writes the logs to disk (ramdisk) and takes care of log rotation
+etc. This is standard s6 stuff, we're not innovating here.
+
+Into the middle of this pipeline we insert a program called
+[logtap](https://gti.telent.net/dan/liminix/src/branch/main/pkgs/logtap/logtap.c)
+which copies its input to its output and also to a fifo - but only
+writes to the fifo if the previous writes worked (i.e. it doesn't back
+up or stop working if the tap is not connected). The standard output from
+logtap goes on to the default logger, so local logging is unaffected -
+which is important if the network is down or hasn't come up yet.
+
+This is a change from last year's version, which used a unix domain
+socket instead of a fifo. Two reasons: first, we need to know which
+messages were sent successfully and which weren't. It was difficult to
+tell reliably and without latency whether there was anything at the
+other end of the socket, whereas we learn almost instantly when a fifo
+write fails. Second, it makes it easier to implement a shipper because
+it can just open the fifo and read from it, instead of having to call
+socket functions.
+
+## Hang a reader on the tap
+
+The log shipper opens the other end of the fifo and ... ships the
+logs. I've chosen
+[VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs)
+(wrapped in an HTTPS reverse proxy) as my centralised log service, so
+my log shipper has to conect with HTTPS to the service endpoint and
+send ["jsonline" log messages](https://docs.victoriametrics.com/victorialogs/data-ingestion/#json-stream-api). In
+fact, my log shipper just speaks pidgin HTTP on [file descriptors 6 and 7](https://cr.yp.to/proto/ucspi.txt) and leverages
+[s6-tlsclient](https://skarnet.org/software/s6-networking/s6-tlsclient.html)
+to do the actual TCP/TLS heavy lifting.
+
+This is all new since last year when we were just splatting raw logs
+over a socket connection instead of doing this fancy JSON stuff. It
+did mean writing a parser for [TAI64N](https://gti.telent.net/dan/liminix/src/branch/main/pkgs/anoia/tai64.fnl) external timestamps and some
+functions to convert it to UTC: as a matter of principle (read:
+stubbornness) I do appreciate that my log message timestamps won't go
+forwards and backwards arbitrarily when leap seconds are decreed, but
+I guess almost nobody else (at least, neither VictoriaLogs nor Zinc)
+thinks it's important.
+
+```nix
+ # in liminix config
+ logging.shipping = {
+ enable = true;
+ command =
+ let certc = config.services.client-cert;
+ in ''
+ export CERTFILE=$(output_path ${certc} certificate)
+ export CAFILE=$(output_path ${certc} ca-certificate)
+ export KEYFILE=$(output_path ${certc} key)
+ ${pkgs.s6-networking}/bin/s6-tlsclient -j -y -k loghost.example.org \
+ 10.0.0.1 443 \
+ ${pkgs.logshippers}/bin/victorialogsend https://loghost.example.org/insert/jsonline
+ '';
+ dependencies = [services.qemu-hyp-route services.client-cert];
+ };
+```
+
+## ... using the TLS cert you previously requested
+
+Before the log shipper can start, it needs to get its TLS client
+certificate, by making a CSR and sending it to
+[Certifix](https://gti.telent.net/dan/certifix). The
+[certifix-client](https://gti.telent.net/dan/liminix/src/branch/main/pkgs/certifix-client/certifix-client.fnl)
+is _almost_ the same as last year's version except that it uses
+[lua-http](https://daurnimator.github.io/lua-http/0.4/) instead of
+[fetch-freebsd](https://github.com/jrmarino/fetch-freebsd) as the http
+interface. This is because last year's version wasn't work
+when asked to traverse the baroque maze of iptables forwarding and
+QEMU Slirp networking that lies between my Liminix test network and my
+VictoriaLogs instance. After a long time staring at pcap dumps I gave
+up trying to work out why and just rewrote that bit.
+
+It's important to have an (at least vaguely) accurate clock before
+attempting HTTPS, because the server certificate has a "not valid
+before" field, so OpenSSL won't like it if you say it's still 1970.
+
+```
+ # in liminix config
+ services.client-cert = svc.tls-certificate.certifix-client.build {
+ caCertificate = builtins.readFile /var/lib/certifix/certs/ca.crt;
+ subject = "C=GB,ST=London,O=Example Org,OU=devices,CN=${config.hostname}";
+ secret = builtins.readFile /var/lib/certifix/challengePassword;
+ serviceUrl = "https://loaclhost.lan:19613/sign";
+ dependencies = [ config.services.ntp ] ;
+ };
+```
+
+
+## ... to connect to an HTTPS reverse proxy
+
+Originally I planned to put a Lets Encrypt cert in front of
+Victorialogs, but that would need 500k of CA certificate bundle on
+each device, which is quite a lot on devices with little flash. So it
+makes more sense to use the Certifix CA here too.
+
+Persuading the OpenSSL command line tools to make a CSR with a
+`challengePassword` was probably as much work as writing something
+with luaossl would have been - it was certainly messier - but the
+point is I didn't know that when I started.
+
+
+```nix
+ # in nixos configuration.nix
+ systemd.services."loghost-certificate" =
+ let
+ dir = "/var/lib/certifix";
+ pw = builtins.readFile "${dir}/private/challengePassword";
+ in {
+ script = ''
+ set -eu
+ cd ${dir}
+ PATH=${pkgs.openssl}/bin:${pkgs.curl}/bin:$PATH
+ openssl req -config <(printf '[req]\nprompt=no\nattributes=attrs\ndistinguished_name=DN\n[DN]"C=GB\nST=London\nO=Example Org\nCN=loghost\n[attrs]\nchallengePassword=${pw}') -newkey rsa:2048 -addext "extendedKeyUsage = serverAuth" -addext "subjectAltName = DNS:loghost.lan,DNS:loghost,DNS:loghost.example.org" -nodes -keyout private/loghost.key --out certs/loghost.csr
+ curl --cacert certs/ca.crt -H 'content-type: application/x-pem-file' --data-binary @certs/loghost.csr https://localhost:19613/sign -o certs/loghost.crt
+ '';
+ serviceConfig = {
+ Type = "oneshot";
+ User = "root";
+ ReadWritePaths = ["/var/lib/certifix"];
+ StateDirectory = "certifix";
+ };
+ startAt = "monthly";
+ };
+```
+
+The proxy itself is just Nginx with `ssl_verify_client` set, but
+certifix-client holds the https connection open so remember to disable
+proxy buffering or you aren't getting your logs in any kind of timely
+fashion.
+
+```nix
+ # in nixos configuration.nix
+ services.nginx.virtualHosts."loghost.example.org" = {
+ forceSSL = true;
+ sslTrustedCertificate = /var/lib/certifix/certs/ca.crt;
+ sslCertificateKey = "/var/lib/certifix/private/loghost.key";
+ sslCertificate = "/var/lib/certifix/certs/loghost.crt";
+
+ extraConfig = ''
+ ssl_verify_client on;
+ proxy_buffering off;
+ proxy_request_buffering off;
+ '';
+
+ locations."/".proxyPass = "http://127.0.0.1:9428/";
+ };
+```
+
+Just as I did last year, I'm going to finish by claiming that this is
+basically finished and it just needs installing on some real
+devices. Hopefully I'm right this time, though.
