From 8ee10214c8c77ac176c4856aa4b2ce60f1f0fdd1 Mon Sep 17 00:00:00 2001 From: Daniel Barlow Date: Sun, 1 Jun 2025 20:50:26 +0100 Subject: [PATCH] async tile fetcher we use cqueues, which is the async framework that lua-http is built on. we integrate it into the glib event loop rather hackily by calling the cqueues event stepper ever 20ms from a glib timeout function overpass has very low rate limits so we handle a 429 response by sleeping for a random length of time and retrying. This is, also, a bit of a hack --- README | 65 +++++++++++++++++++++++---------------------- pkgs/maps/main.fnl | 14 +++++++++- pkgs/maps/tiles.fnl | 62 +++++++++++++++++++++++++++--------------- 3 files changed, 86 insertions(+), 55 deletions(-) diff --git a/README b/README index dec6e66..44c54b4 100644 --- a/README +++ b/README @@ -19,8 +19,6 @@ write the app in fennel. I want it to - show where I am on a map - record trail of where I've been (note: indoor counts too) - - can we somehow do non-flakey bluetooth (is it dbus?) @@ -88,11 +86,6 @@ elapsed time: what should it actually show? moving time, I guess should we rename bearing as course in nmea? -rotating the map is going to be complicated because the widget we're -using doesn't support it (bitmapped map tiles) - - - perhaps we need a server-side component for route planning @@ -106,9 +99,9 @@ we can't rotate the map using OsmGpsMap widget because the labels will be sideways or upside down, so we need something with vectors that we can rotate -a) we can get data from overpass api as json +[done] a) we can get data from overpass api as json -b) we would like to cache the results, which means some kind of +[done badly] b) we would like to cache the results, which means some kind of chunking or tiling so that the json for position a is the same as the json for position b. @@ -119,37 +112,45 @@ do it by hand - - minor roads - major roads -or so something smart but complicated like "only return ways that +or do something smart but complicated like "only return ways that cover more than 1/16th the length of the tile" -d) I think we will need some kind of server so that multiple users get -the benefit of the caching. If we're going to do that, should it also -do transformation e.g. from lat/long to x/y co-ordinates? We don't -need this bit yet though +d) render ways according to their type (road/cycleway/path/etc) +e) label the ways + +f) async tile fetching + +we don't want everything to stop when it's time to fetch a new +row of tiles, what are our options? lua-http is built on cqueues +which is async enough to make my head hurt, but we also need +to make it coexist with the gtk event loop + +assumptions: +1) gtk stuff has to happen in the main thread (whatever that is...) +so we can't control it from cqueues because that has its own +threading stuff +2) there will be lots of fds from lua-http, do we really want the +housekeeping of making GLib.io_add_watch for each of them? it looks +like adding a glib source from lgi is not currently practical +https://github.com/lgi-devs/lgi/issues/111 + +3) if we put http calls inside cq:wrap, that make them background +provided that we call (cq:step 0) +periodically. we could do that in a glib idle function, perhaps. + +- The tile fetcher would need to know where to write the data when + eventually it comes back +- need some say to not fetch the same tile 18 times if there's more than + one request for it while a previous request is in progress + + +---- https://git.syndicate-lang.org/tonyg/squeak-phone/raw/commit/474960ddc665ed445a1f5afb0164fe39057720f9/devices/pine64-pinephone/modem-docs/80545ST10798A_LM940_QMI_Command_Reference_Guide_r3.pdf ---- -we need to extend to multiple tiles'-worth of map - - -* get tile for curent lat/long and request overpass data for enough - surrounding tiles to fill the screen - -* I think a way is served with all its nodes whether or not they're in -the bbox, so we can just store the ids of ways we've seen and skip -them if the come up again - -* render all the polylines into the widget (some day also the labels etc) - -* to get it centred on the cyclist, take the tile fractional part * - 256, and translate the canvas up and left by that amount - -* add a cache of [x,y,z] -> polylines so that we don't keep hitting overpass - - diff --git a/pkgs/maps/main.fnl b/pkgs/maps/main.fnl index 2c6f418..73db50c 100644 --- a/pkgs/maps/main.fnl +++ b/pkgs/maps/main.fnl @@ -1,5 +1,7 @@ ; (local { : view } (require :fennel)) (local { : fdopen } (require :posix.stdio)) +(local cqueues (require :cqueues)) + (local nmea (require :nmea)) (local tiles (require :tiles)) @@ -87,6 +89,8 @@ label.readout { : num-tiles-x : num-tiles-y })) +(local cq (cqueues.new)) + (fn cairo-the-map [window] (let [{ : lat : lon : zoom } app-state { : num-tiles-x : num-tiles-y &as bounds } (map-bounds lat lon zoom) @@ -94,7 +98,7 @@ label.readout { (for [x bounds.min.x bounds.max.x] (for [y bounds.min.y bounds.max.y] - (merge lines (tiles.polylines x y zoom)))) + (merge lines (tiles.polylines cq x y zoom)))) (let [map-surface (window:create_similar_surface @@ -270,6 +274,14 @@ label.readout { (GLib.io_add_watch channel 0 events #(read-gnss handle))) +(GLib.timeout_add + GLib.PRIORITY_DEFAULT + 20 ; ms + (fn [] + (cq:step 0) + true) + nil nil) + (window:add (doto (Gtk.Overlay {}) (: :add (osm-widget)) diff --git a/pkgs/maps/tiles.fnl b/pkgs/maps/tiles.fnl index f6351c1..7a4f306 100644 --- a/pkgs/maps/tiles.fnl +++ b/pkgs/maps/tiles.fnl @@ -1,6 +1,7 @@ (local req (require :http.request)) (local { : dict_to_query } (require :http.util)) (local json (require :json)) +(local cqueues (require :cqueues)) (import-macros { : define-tests : expect : expect= : expect-near } :assert) (local { : view } (require :fennel)) @@ -81,19 +82,8 @@ f (do (f:close) true) _ false)) -(fn unparsed-from-disk [x y zoom fetch-fn] - (let [k (.. x "_" y "_" zoom) - pathname (.. "/tmp/tiles/" k ".json")] - (if (file-exists? pathname) - (with-open [i (io.open pathname :r)] - (i:read "*a")) - (with-open [j (io.open pathname :w)] - (let [g (fetch-fn)] - (j:write g) - g))))) - (fn unparsed-for-xyz [x y zoom] - (let [(lat lon) (tile->latlon x y zoom) + (let [(lat lon) (tile->latlon x y zoom) o (overpass lat lon zoom) r (req.new_from_uri @@ -102,16 +92,44 @@ (tset r.headers ":method" "POST") (r:set_body (dict_to_query query)) (let [(headers stream) (r:go)] - (stream:get_body_as_string)))) + (if (= (headers:get ":status") "429") + nil + (stream:get_body_as_string))))) -(fn polylines-from-net [x y zoom] - (let [s (unparsed-from-disk - x y zoom - (fn [] - (unparsed-for-xyz x y zoom))) - ;_ (print :unoparsed (s:sub 1 40)) - data (json.decode s)] - (canvas data.elements))) +;; if we have json in disk, return it +;; if we have an empty file on disk, that signifies a request in +;; flight, so return a "pending" sentinel +;; if we have no disk file, kick off a request and send "pending" sentinel + +;; we'd like to have a way for completed background fetch to signal +;; so that the map can be redrawn + +(fn polylines [cq x y zoom] + (let [k (.. x "_" y "_" zoom) + pathname (.. "/tmp/tiles/" k ".json")] + (if (file-exists? pathname) + (let [data (with-open [i (io.open pathname :r)] (i:read "*a"))] + (if (= data "") + [] + (canvas (. (json.decode data) :elements)))) + (let [out (io.open pathname :w)] + (cq:wrap (fn [] + (print "getting " k) + (var json nil) + (with-open [f out] + (while (not json) + (set json (unparsed-for-xyz x y zoom)) + (when (not json) + (print "sleeping " k) + (cqueues.sleep (math.random 2 6)))) + (print "got " k) + (f:write json) + true))) + [] ; return no lines for now + )))) -{ :polylines polylines-from-net : latlon->tile } + + + +{ : polylines : latlon->tile }