async tile fetcher

we use cqueues, which is the async framework that lua-http is built
on. we integrate it into the glib event loop rather hackily by calling
the cqueues event stepper ever 20ms from a glib timeout function

overpass has very low rate limits so we handle a 429 response by
sleeping for a random length of time and retrying. This is, also,
a bit of a hack

README

@@ -19,8 +19,6 @@ write the app in fennel. I want it to
 - show where I am on a map
 - record trail of where I've been (note: indoor counts too)
 
-
-
 can we somehow do non-flakey bluetooth (is it dbus?)
 
 
@@ -88,11 +86,6 @@ elapsed time: what should it actually show? moving time, I guess
 
 should we rename bearing as course in nmea?
 
-rotating the map is going to be complicated because the widget we're
-using doesn't support it (bitmapped map tiles)
-
-
-
 perhaps we need a server-side component for route planning
 
 
@@ -106,9 +99,9 @@ we can't rotate the map using OsmGpsMap widget because the labels will
 be sideways or upside down, so we need something with vectors that we
 can rotate
 
-a) we can get data from overpass api as json
+[done] a) we can get data from overpass api as json
 
-b) we would like to cache the results, which means some kind of
+[done badly] b) we would like to cache the results, which means some kind of
 chunking or tiling so that the json for position a is the same as the
 json for position b.
 
@@ -119,37 +112,45 @@ do it by hand -
 - minor roads
 - major roads
 
-or so something smart but complicated like "only return ways that
+or do something smart but complicated like "only return ways that
 cover more than 1/16th the length of the tile"
 
-d) I think we will need some kind of server so that multiple users get
+d) render ways according to their type (road/cycleway/path/etc)
-the benefit of the caching. If we're going to do that, should it also
-do transformation e.g. from lat/long to x/y co-ordinates? We don't
-need this bit yet though
 
+e) label the ways
+
+f) async tile fetching
+
+we don't want everything to stop when it's time to fetch a new
+row of tiles, what are our options? lua-http is built on cqueues
+which is async enough to make my head hurt, but we also need
+to make it coexist with the gtk event loop
+
+assumptions:
+1) gtk stuff has to happen in the main thread (whatever that is...)
+so we can't control it from cqueues because that has its own
+threading stuff
+2) there will be lots of fds from lua-http, do we really want the
+housekeeping of making GLib.io_add_watch for each of them? it looks
+like adding a glib source from lgi is not currently practical
+https://github.com/lgi-devs/lgi/issues/111
+
+3) if we put http calls inside cq:wrap, that make them background
+provided that we call (cq:step 0)
+periodically. we could do that in a glib idle function, perhaps.
+
+- The tile fetcher would need to know where to write the data when
+  eventually it comes back
+- need some say to not fetch the same tile 18 times if there's more than
+  one request for it while a previous request is in progress
+
+
+---- 
 
 https://git.syndicate-lang.org/tonyg/squeak-phone/raw/commit/474960ddc665ed445a1f5afb0164fe39057720f9/devices/pine64-pinephone/modem-docs/80545ST10798A_LM940_QMI_Command_Reference_Guide_r3.pdf
 
 ----
 
 
-we need to extend to multiple tiles'-worth of map
-
-
-* get tile for curent lat/long and request overpass data for enough
-  surrounding tiles to fill the screen
-
-* I think a way is served with all its nodes whether or not they're in
-the bbox, so we can just store the ids of ways we've seen and skip
-them if the come up again
-
-* render all the polylines into the widget (some day also the labels etc)
-
-* to get it centred on the cyclist, take the tile fractional part *
-  256, and translate the canvas up and left by that amount
-
-* add a cache of [x,y,z] -> polylines so that we don't keep hitting overpass
-
-
 
 

pkgs/maps/main.fnl

@@ -1,5 +1,7 @@
 ; (local { : view } (require :fennel))
 (local { : fdopen } (require :posix.stdio))
+(local cqueues (require :cqueues))
+
 (local nmea (require :nmea))
 (local tiles (require :tiles))
 
@@ -87,6 +89,8 @@ label.readout {
      : num-tiles-x : num-tiles-y
      }))
 
+(local cq (cqueues.new))
+
 (fn cairo-the-map [window]
   (let [{ : lat : lon : zoom } app-state
         { : num-tiles-x : num-tiles-y &as bounds } (map-bounds lat lon zoom)
@@ -94,7 +98,7 @@ label.readout {
 
     (for [x bounds.min.x bounds.max.x]
       (for [y bounds.min.y bounds.max.y]
-        (merge lines (tiles.polylines x y zoom))))
+        (merge lines (tiles.polylines cq x y zoom))))
 
     (let [map-surface
           (window:create_similar_surface
@@ -270,6 +274,14 @@ label.readout {
   (GLib.io_add_watch channel 0 events #(read-gnss handle)))
 
 
+(GLib.timeout_add
+ GLib.PRIORITY_DEFAULT
+ 20 ; ms
+ (fn []
+   (cq:step 0)
+   true)
+ nil nil)
+
 (window:add
  (doto (Gtk.Overlay {})
    (: :add (osm-widget))

pkgs/maps/tiles.fnl

@@ -1,6 +1,7 @@
 (local req (require :http.request))
 (local { : dict_to_query } (require :http.util))
 (local json (require :json))
+(local cqueues (require :cqueues))
 
 (import-macros { : define-tests : expect : expect= : expect-near } :assert)
 (local { : view } (require :fennel))
@@ -81,19 +82,8 @@
     f (do (f:close) true)
     _ false))
 
-(fn unparsed-from-disk [x y zoom fetch-fn]
-  (let [k (..  x "_" y "_" zoom)
-        pathname (.. "/tmp/tiles/" k ".json")]
-    (if (file-exists? pathname)
-        (with-open [i (io.open pathname :r)]
-          (i:read "*a"))
-        (with-open [j (io.open pathname :w)]
-          (let [g (fetch-fn)]
-            (j:write g)
-            g)))))
-
 (fn unparsed-for-xyz [x y zoom]
-    (let [(lat lon) (tile->latlon x y zoom)
+  (let [(lat lon) (tile->latlon x y zoom)
         o (overpass lat lon zoom)
         r
         (req.new_from_uri
@@ -102,16 +92,44 @@
     (tset r.headers ":method" "POST")
     (r:set_body (dict_to_query query))
     (let [(headers stream) (r:go)]
-      (stream:get_body_as_string))))
+      (if (= (headers:get  ":status") "429")
+          nil
+          (stream:get_body_as_string)))))
 
-(fn polylines-from-net [x y zoom]
+;; if we have json in disk, return it
-  (let [s (unparsed-from-disk
+;; if we have an empty file on disk, that signifies a request in
-           x y zoom
+;;   flight, so return a "pending" sentinel
-           (fn []
+;; if we have no disk file, kick off a request and send "pending" sentinel
-             (unparsed-for-xyz x y zoom)))
+
-        ;_ (print :unoparsed (s:sub 1 40))
+;; we'd like to have a way for completed background fetch to signal
-        data (json.decode s)]
+;; so that the map can be redrawn
-    (canvas data.elements)))
+
+(fn polylines [cq x y zoom]
+  (let [k (..  x "_" y "_" zoom)
+        pathname (.. "/tmp/tiles/" k ".json")]
+    (if (file-exists? pathname)
+        (let [data (with-open [i (io.open pathname :r)] (i:read "*a"))]
+          (if (= data "")
+              []
+              (canvas (. (json.decode data) :elements))))
+        (let [out (io.open pathname :w)]
+          (cq:wrap (fn []
+                     (print "getting " k)
+                     (var json nil)
+                     (with-open [f out]
+                       (while (not json)
+                         (set json (unparsed-for-xyz x y zoom))
+                         (when (not json)
+                           (print "sleeping " k)
+                           (cqueues.sleep (math.random 2 6))))
+                       (print "got " k)
+                       (f:write json)
+                       true)))
+          [] ; return no lines for now
+          ))))
 
 
-{ :polylines polylines-from-net : latlon->tile }
+
+
+
+{ : polylines : latlon->tile }