commit 7c18f4442b43514972070d23937038d37a5bf0f6
parent dfbf6ac91906a1baf5687be0440c2acd243430ce
Author: Daniel Barlow <dan@telent.net>
Date: Sun, 1 Jun 2025 20:50:26 +0100
async tile fetcher
we use cqueues, which is the async framework that lua-http is built
on. we integrate it into the glib event loop rather hackily by calling
the cqueues event stepper ever 20ms from a glib timeout function
overpass has very low rate limits so we handle a 429 response by
sleeping for a random length of time and retrying. This is, also,
a bit of a hack
Diffstat:
3 files changed, 81 insertions(+), 50 deletions(-)
diff --git a/README b/README
@@ -19,8 +19,6 @@ write the app in fennel. I want it to
- show where I am on a map
- record trail of where I've been (note: indoor counts too)
-
-
can we somehow do non-flakey bluetooth (is it dbus?)
@@ -88,11 +86,6 @@ elapsed time: what should it actually show? moving time, I guess
should we rename bearing as course in nmea?
-rotating the map is going to be complicated because the widget we're
-using doesn't support it (bitmapped map tiles)
-
-
-
perhaps we need a server-side component for route planning
@@ -106,9 +99,9 @@ we can't rotate the map using OsmGpsMap widget because the labels will
be sideways or upside down, so we need something with vectors that we
can rotate
-a) we can get data from overpass api as json
+[done] a) we can get data from overpass api as json
-b) we would like to cache the results, which means some kind of
+[done badly] b) we would like to cache the results, which means some kind of
chunking or tiling so that the json for position a is the same as the
json for position b.
@@ -119,36 +112,44 @@ do it by hand -
- minor roads
- major roads
-or so something smart but complicated like "only return ways that
+or do something smart but complicated like "only return ways that
cover more than 1/16th the length of the tile"
-d) I think we will need some kind of server so that multiple users get
-the benefit of the caching. If we're going to do that, should it also
-do transformation e.g. from lat/long to x/y co-ordinates? We don't
-need this bit yet though
+d) render ways according to their type (road/cycleway/path/etc)
+e) label the ways
-https://git.syndicate-lang.org/tonyg/squeak-phone/raw/commit/474960ddc665ed445a1f5afb0164fe39057720f9/devices/pine64-pinephone/modem-docs/80545ST10798A_LM940_QMI_Command_Reference_Guide_r3.pdf
-
-----
+f) async tile fetching
+we don't want everything to stop when it's time to fetch a new
+row of tiles, what are our options? lua-http is built on cqueues
+which is async enough to make my head hurt, but we also need
+to make it coexist with the gtk event loop
-we need to extend to multiple tiles'-worth of map
+assumptions:
+1) gtk stuff has to happen in the main thread (whatever that is...)
+so we can't control it from cqueues because that has its own
+threading stuff
+2) there will be lots of fds from lua-http, do we really want the
+housekeeping of making GLib.io_add_watch for each of them? it looks
+like adding a glib source from lgi is not currently practical
+https://github.com/lgi-devs/lgi/issues/111
+3) if we put http calls inside cq:wrap, that make them background
+provided that we call (cq:step 0)
+periodically. we could do that in a glib idle function, perhaps.
-* get tile for curent lat/long and request overpass data for enough
- surrounding tiles to fill the screen
+- The tile fetcher would need to know where to write the data when
+ eventually it comes back
+- need some say to not fetch the same tile 18 times if there's more than
+ one request for it while a previous request is in progress
-* I think a way is served with all its nodes whether or not they're in
-the bbox, so we can just store the ids of ways we've seen and skip
-them if the come up again
-* render all the polylines into the widget (some day also the labels etc)
+----
-* to get it centred on the cyclist, take the tile fractional part *
- 256, and translate the canvas up and left by that amount
+https://git.syndicate-lang.org/tonyg/squeak-phone/raw/commit/474960ddc665ed445a1f5afb0164fe39057720f9/devices/pine64-pinephone/modem-docs/80545ST10798A_LM940_QMI_Command_Reference_Guide_r3.pdf
-* add a cache of [x,y,z] -> polylines so that we don't keep hitting overpass
+----
diff --git a/pkgs/maps/main.fnl b/pkgs/maps/main.fnl
@@ -1,5 +1,7 @@
; (local { : view } (require :fennel))
(local { : fdopen } (require :posix.stdio))
+(local cqueues (require :cqueues))
+
(local nmea (require :nmea))
(local tiles (require :tiles))
@@ -87,6 +89,8 @@ label.readout {
: num-tiles-x : num-tiles-y
}))
+(local cq (cqueues.new))
+
(fn cairo-the-map [window]
(let [{ : lat : lon : zoom } app-state
{ : num-tiles-x : num-tiles-y &as bounds } (map-bounds lat lon zoom)
@@ -94,7 +98,7 @@ label.readout {
(for [x bounds.min.x bounds.max.x]
(for [y bounds.min.y bounds.max.y]
- (merge lines (tiles.polylines x y zoom))))
+ (merge lines (tiles.polylines cq x y zoom))))
(let [map-surface
(window:create_similar_surface
@@ -270,6 +274,14 @@ label.readout {
(GLib.io_add_watch channel 0 events #(read-gnss handle)))
+(GLib.timeout_add
+ GLib.PRIORITY_DEFAULT
+ 20 ; ms
+ (fn []
+ (cq:step 0)
+ true)
+ nil nil)
+
(window:add
(doto (Gtk.Overlay {})
(: :add (osm-widget))
diff --git a/pkgs/maps/tiles.fnl b/pkgs/maps/tiles.fnl
@@ -1,6 +1,7 @@
(local req (require :http.request))
(local { : dict_to_query } (require :http.util))
(local json (require :json))
+(local cqueues (require :cqueues))
(import-macros { : define-tests : expect : expect= : expect-near } :assert)
(local { : view } (require :fennel))
@@ -81,19 +82,8 @@
f (do (f:close) true)
_ false))
-(fn unparsed-from-disk [x y zoom fetch-fn]
- (let [k (.. x "_" y "_" zoom)
- pathname (.. "/tmp/tiles/" k ".json")]
- (if (file-exists? pathname)
- (with-open [i (io.open pathname :r)]
- (i:read "*a"))
- (with-open [j (io.open pathname :w)]
- (let [g (fetch-fn)]
- (j:write g)
- g)))))
-
(fn unparsed-for-xyz [x y zoom]
- (let [(lat lon) (tile->latlon x y zoom)
+ (let [(lat lon) (tile->latlon x y zoom)
o (overpass lat lon zoom)
r
(req.new_from_uri
@@ -102,16 +92,44 @@
(tset r.headers ":method" "POST")
(r:set_body (dict_to_query query))
(let [(headers stream) (r:go)]
- (stream:get_body_as_string))))
+ (if (= (headers:get ":status") "429")
+ nil
+ (stream:get_body_as_string)))))
-(fn polylines-from-net [x y zoom]
- (let [s (unparsed-from-disk
- x y zoom
- (fn []
- (unparsed-for-xyz x y zoom)))
- ;_ (print :unoparsed (s:sub 1 40))
- data (json.decode s)]
- (canvas data.elements)))
+;; if we have json in disk, return it
+;; if we have an empty file on disk, that signifies a request in
+;; flight, so return a "pending" sentinel
+;; if we have no disk file, kick off a request and send "pending" sentinel
+;; we'd like to have a way for completed background fetch to signal
+;; so that the map can be redrawn
-{ :polylines polylines-from-net : latlon->tile }
+(fn polylines [cq x y zoom]
+ (let [k (.. x "_" y "_" zoom)
+ pathname (.. "/tmp/tiles/" k ".json")]
+ (if (file-exists? pathname)
+ (let [data (with-open [i (io.open pathname :r)] (i:read "*a"))]
+ (if (= data "")
+ []
+ (canvas (. (json.decode data) :elements))))
+ (let [out (io.open pathname :w)]
+ (cq:wrap (fn []
+ (print "getting " k)
+ (var json nil)
+ (with-open [f out]
+ (while (not json)
+ (set json (unparsed-for-xyz x y zoom))
+ (when (not json)
+ (print "sleeping " k)
+ (cqueues.sleep (math.random 2 6))))
+ (print "got " k)
+ (f:write json)
+ true)))
+ [] ; return no lines for now
+ ))))
+
+
+
+
+
+{ : polylines : latlon->tile }
