v5

APLSource/ADMIN/ReadCache.aplf

@@ -0,0 +1,24 @@
+ r←ReadCache file;cache;tie
+⍝ Load a persisted cache so a cold start serves the previous refresh's data immediately.
+⍝ file: component file path                       :: simple character vector
+⍝    r: what happened, for the startup transcript :: simple character vector
+ :If ~⎕NEXISTS file
+     r←'No cache file at ',file,'. One will be written when the cache is next built.'
+     :Return
+ :EndIf
+ r←'Cache file could not be used; waiting on the background build instead.'
+ :Trap 0
+     tie←file ⎕FTIE 0
+     cache←⎕FREAD tie 1
+     ⎕FUNTIE tie
+     :If #.DCMS.CACHE.Valid cache
+         #.DCMS.CACHE.Rehydrate cache
+         #.DCMS.GLOBAL.cache←cache   ⍝ Single-line update is safe across threads. Readers take snapshots to get complete cache view.
+         r←'Loaded cache written at ',(⊃##.DT.ISODateTime 13 ⎕NINFO file),' UTC'
+     :Else
+         ⎕←'Cache file ',file,' is not the shape this version of DCMS expects.'
+     :EndIf
+ :Else
+     ⎕←'Could not read ',file,'. ',⎕DMX.(EM,': ',Message)
+     {0::⍬ ⋄ ⎕FUNTIE tie}⍬            ⍝ tie is unassigned if ⎕FTIE was what failed
+ :EndTrap

APLSource/ADMIN/Refresh.aplf

@@ -1,16 +1,47 @@
- r←Refresh req
-⍝ Stark endpoint function, argument is request object or spec sigil
+ r←Refresh req;msg;owned;responses;state
+⍝ Stark endpoint function, argument is request object, timer object, or spec sigil
  :If '⍠SPEC'≡req
+     responses←##.SPEC.responses
      r←(
          tags:,⊂'Admin' ⋄ summary:'Refresh cached API data' ⋄ operationId:'refreshData'
-         description:'Fetches external API data, then rebuilds the cached joined views served by the public read endpoints.'
+         description:'Fetches external API data, then rebuilds the cached joined views served by the public read endpoints. Returns immediately; the refresh runs in the background. At most one refresh runs at a time, with at most one queued behind it.'
+         responses:(
+             ⍙202:(
+                 description:'Request accepted. A refresh was started, or queued behind the one already running.'
+                 content:(⍙application⍙47⍙json:(schema:(
+                     type:'object'
+                     required:('message' ⋄ 'monitorUrl')
+                     properties:(
+                         message:(type:'string' ⋄ description:'Whether the refresh started immediately or was queued behind a running one.')
+                         monitorUrl:(type:'string' ⋄ format:'uri' ⋄ description:'Status endpoint for monitoring this refresh.')
+                     )
+                 )))
+             )
+             ⍙401:responses.unauthorized
+         )
      )
      :Return
  :EndIf
- :Trap 202 404
-     ##.IMPORT.YOUTUBE.RefreshData''
+ state←##.GLOBAL.refresh
+⍝ Take the refresh serialisation lock: only one thread may rebuild the cache at a time.
+ :Hold '#.DCMS.GLOBAL.refresh'
+     :If state.running∧state.thread∊⎕TNUMS
+         state.pending←1              ⍝ At most one pending job; a further trigger is a no-op
+         owned←0
+     :Else
+         state.running←1
+         state.thread←RunRefresh&⍬    ⍝ the worker thread ID
+         state.pending←0
+         owned←1
+     :EndIf
+ :EndHold
+ :If owned
+     msg←'Request accepted. Refresh task started. View status at monitorUrl'
  :Else
-     ⎕←'DATA NOT REFRESHED. ',⎕DMX.Message
- :EndTrap
- RefreshData&1   ⍝ Trigger cache build in new thread, so GET /admin/refresh can return immediately
- ⎕←r←'Refreshed data at ',⍕⎕TS
+     msg←'Request accepted. One refresh task is pending and will begin once the current task is finished. View status at monitorUrl'
+ :EndIf
+ ##.Log'[REFRESH] ',msg
+ r←(message:msg ⋄ monitorUrl:(##.GetEnv'PUBLIC_URL'),'/admin/refresh/status')
+ :If 9=⎕NC'req'                       ⍝ Only a Jarvis request object can carry a status
+     'Accepted'req.SetStatus 202       ⍝ Jarvis has no 202 in HttpStatus, so name it explicitly
+ :EndIf

APLSource/ADMIN/RefreshData.aplf

@@ -1,26 +1,43 @@
- r←RefreshData opt
-⍝ opt: whether to build cache in isolate or main process :: Boolean
-⍝   r: information about when data refresh was triggered :: simple character vector
+ {new}←RefreshData;build_opt;result;⍙CACHE;Content;Copy
+⍝ new: newly built cache namespace :: namespace
+ ##.Log'[REFRESH] Importing from YouTube'
  :Trap 202 404
      ##.IMPORT.YOUTUBE.RefreshData''
  :Else
-     ⎕←'DATA NOT REFRESHED. ',⎕DMX.Message
+     ⎕←'[REFRESH] DATA NOT REFRESHED. ',⎕DMX.Message
  :EndTrap
+ ##.Log'[REFRESH] Imported from YouTube'
  build_opt←(
      using:##.⎕USING
      app_dir:##.app_dir
  )
- :If opt
-     ⍙CACHE←#.ø #.DCMS.CACHE         ⍝ Spawn isolate to build cache in separate process
-     result←⍙CACHE.Build build_opt   ⍝ Build new cache in isolate. Does not block threads in main process despite waiting for future to resolve.
-     ##.Log'⍙CACHE isolate running:',#.isolate.Running'result'
-     'new'⎕NS result                 ⍝ Must deep clone 'new'⎕NS ref because result ref from namespace-returning isolate cannot be dotted into (interpreter bug)
- :Else
-     new←#.DCMS.CACHE.Build build_opt
+ ##.Log'[REFRESH] Building cache'
+ ⍙CACHE←#.ø #.DCMS.CACHE         ⍝ Spawn isolate to build cache in separate process
+ result←⍙CACHE.Build build_opt   ⍝ Build new cache in isolate. Does not block threads in main process despite waiting for future to resolve.
+ ##.Log'[REFRESH] ⍙CACHE isolate running:',#.isolate.Running'result'
+ 'new'⎕NS result                 ⍝ Must deep clone 'new'⎕NS ref because result ref from namespace-returning isolate cannot be dotted into (interpreter bug)
+ ##.Log'[REFRESH] Built cache'
+⍝ Replace sub-namespace references with local copies, so the cache is self-contained and can be written to a component file.
+ Copy←{0=≢⍵:⍵ ⋄ ⎕NS¨⍵}
+ new.presenters←Copy new.presenters
+ new.events.data←Copy new.events.data
+ new.videos←⎕NS new.videos
+
+ Content←{1(220⌶)(⍵.videos.(fields values))(⍵.events.data)(⍵.events.has_videos)(⍵.presenters)}
+ new.last_updated←1 ⎕DT'Z'
+ :If 0≠#.DCMS.GLOBAL.⎕NC'cache'   ⍝ Cold start has no cache to compare against
+ :AndIf (Content new)≡Content #.DCMS.GLOBAL.cache
+     new.last_updated←#.DCMS.GLOBAL.cache.last_updated   ⍝ Unchanged content keeps its ETag
  :EndIf
 
- :Hold '#.DCMS.GLOBAL.cache'   ⍝ Single-line update is safe across threads. Readers take snapshots to get complete cache view.
-     #.DCMS.GLOBAL.cache←new ⋄ #.DCMS.GLOBAL.cache.videos.(Lookup←index∘⍳)
- :EndHold
+ new.etag←'W/"','"',⍨'0'@(' '∘=)∊4 2 2 2 2 2 3-⍛↑∘⍕¨⊃1 ¯1 ⎕DT new.last_updated
+ new.last_updated←¯1 1 ⎕DT⊂6↑⊃1 ¯1 ⎕DT new.last_updated   ⍝ Only second precision for HTTP-date
+
+ new←#.DCMS.CACHE.Rehydrate new        ⍝ Hash the index and bind the lookup function over it
+
+ #.DCMS.GLOBAL.cache←new               ⍝ Single-line update is safe across threads. Readers take snapshots to get complete cache view.
+
+⍝ Persist the cache we are now serving, so the next cold start can serve it immediately.
+ (##.GetEnv'CACHE_FILE')WriteCache new
 
- ⎕←r←'Refreshed data at ',⍕⎕TS
+ ##.Log'[REFRESH] Refreshed #.DCMS.GLOBAL.cache'

APLSource/ADMIN/RefreshStatus.aplf

@@ -0,0 +1,31 @@
+ r←RefreshStatus req;responses;ts
+⍝ GET /admin/refresh/status — current and most recent data refresh state
+ :If req≡'⍠SPEC'
+     responses←##.SPEC.responses
+     ts←{(type:'string' ⋄ format:'date-time' ⋄ nullable:⊂'true' ⋄ description:⍵)}
+     r←(
+         tags:,⊂'Admin' ⋄ summary:'Get data refresh status' ⋄ operationId:'getRefreshStatus'
+         responses:(
+             ⍙200:(
+                 description:'Current and most recent data refresh state'
+                 content:(⍙application⍙47⍙json:(schema:(
+                     type:'object'
+                     properties:(
+                         status:(type:'string' ⋄ enum:('idle' 'running') ⋄ description:'Whether a refresh is running now.')
+                         pending:(type:'integer' ⋄ enum:(0 1) ⋄ description:'1 if a refresh is queued behind the running one.')
+                         started_at:ts'When the current or most recent refresh began; null if none has run.'
+                         finished_at:ts'When the most recent refresh ended; null while one is running.'
+                         last_success_at:ts'When a refresh most recently completed without error.'
+                         last_error_at:ts'When a refresh most recently failed.'
+                         last_error_message:(type:'string' ⋄ description:'Message from the most recent failure; empty if the last refresh succeeded.')
+                     )
+                 )))
+             )
+             ⍙401:responses.unauthorized
+         )
+     )
+     :Return
+ :EndIf
+ 'r'⎕NS ##.GLOBAL.refresh                      ⍝ Deep copy: a swap mid-serialisation must not tear the response
+ r.status←(1+r.running)⊃'idle' 'running'
+ r.⎕EX'running' 'thread'                       ⍝ Internal fields are not part of the API

APLSource/ADMIN/RunRefresh.aplf

@@ -0,0 +1,33 @@
+ RunRefresh dummy;ISODateTime;again;state
+⍝ Serialised refresh worker. Spawned with & by Refresh.
+⍝ Holds the refresh serialisation lock. If the pending flag
+⍝ is set by the end of a refresh, this function loops and refreshes again.
+⍝ dummy: ignored
+ ISODateTime←#.DCMS.DT.ISODateTime
+ state←##.GLOBAL.refresh
+ :Repeat
+     ##.Log'[REFRESH] Cycle start on thread ',⍕⎕TID
+     state.pending←0                  ⍝ This cycle satisfies any trigger queued so far
+     state.started_at←⊃ISODateTime 1⎕DT'Z'
+     state.finished_at←⊂'null'
+    ⍝ Blanket trap: this thread has no request and no OnErrorFn, so an escaping error
+    ⍝ would hold the lock forever.
+     :Trap 0
+         RefreshData
+         state.last_success_at←⊃ISODateTime 1⎕DT'Z'
+         state.last_error_message←''
+     :Else
+         state.last_error_at←⊃ISODateTime 1⎕DT'Z'
+         state.last_error_message←⎕DMX.(EM,': ',Message)
+         ⎕←'[REFRESH] ',state.last_error_message,∊(⎕UCS 10),¨1↓⎕DMX.DM
+     :EndTrap
+     state.finished_at←⊃ISODateTime 1⎕DT'Z'
+     :Hold '#.DCMS.GLOBAL.refresh'
+         :If again←state.pending
+             state.pending←0
+         :Else
+             state.(running thread)←0 ¯1
+         :EndIf
+     :EndHold
+     ##.Log'[REFRESH] Cycle end, again=',⍕again
+ :Until ~again

APLSource/ADMIN/WPPush.aplf

@@ -20,6 +20,10 @@
                  )))
              )
              ⍙401:responses.unauthorized
+             ⍙502:(
+                 description:'WordPress rejected the request or was unreachable. Signalled by WP.Post.'
+                 content:(⍙application⍙47⍙json:(schema:##.SPEC.schemas.error))
+             )
          )
      )
      :Return

APLSource/ADMIN/WriteCache.aplf

@@ -0,0 +1,23 @@
+ file WriteCache cache;tie;tmp
+⍝ Persist the cache namespace so the next cold start can serve immediately.
+⍝  file: component file path            :: simple character vector
+⍝ cache: self-contained cache namespace :: namespace ref
+ tmp←file,'.tmp'
+ ##.Log'[REFRESH] Writing cache to ',file
+ :Trap 0
+     :If ⎕NEXISTS tmp
+         {}⎕NDELETE tmp    ⍝ Discard a temporary file left by an interrupted write
+     :EndIf
+     tie←tmp ⎕FCREATE 0    ⍝ Create temporary file and move so reader cannot read half-written file
+     cache ⎕FAPPEND tie
+     ⎕FUNTIE tie 
+     :If ⎕NEXISTS file     ⍝ ⎕NMOVE does not overwrite.
+         {}⎕NDELETE file
+     :EndIf
+     {}file ⎕NMOVE tmp
+     ##.Log'[REFRESH] Wrote cache to ',file
+ :Else
+     ⎕←'Could not write ',file,'. Cache not persisted. ',⎕DMX.(EM,': ',Message)
+     {0::⍬ ⋄ ⎕FUNTIE tie}⍬             ⍝ tie is unassigned if ⎕FCREATE failed
+     {0::⍬ ⋄ {}⎕NDELETE tmp}⍬
+ :EndTrap

APLSource/AUTH/ControlAccess.aplf

@@ -6,10 +6,13 @@
 ⍝   authed prefix → require valid x-api-key
 ⍝   neither       → 404 unregistered routes
      public←⊆'/version' '/videos' '/presenters' '/events' '/openapi.json' '/teapot' '/swagger/'
-     authed←⊆'/crud' '/schema' '/admin' '/teapot'
+     authed←⊆'/crud' '/schema' '/admin'
      '/',⍛≡⍵.Endpoint:0                          ⍝ Swagger UI home page at root (Endpoint is ,'/' or ''); exact, not a prefix — '/' as a prefix would open everything
      ∨/public(⊃⍷)¨⊂⍵.Endpoint:0
      ∨/authed(⊃⍷)¨⊂⍵.Endpoint:r⊣##.Log'[AUTH] ',('allow' 'deny'⊃⍨1+r←VerifyToken ⍵),' ',⍵.Endpoint
-     ⍵.Response.Payload←(error:404 ⋄ message:'Not found')
+     payload←(error:404 ⋄ message:'Not found')   ⍝ Pre-routing, so HandleError cannot build this one
+     payload.request←(method:⍵.Method ⋄ endpoint:⍵.Endpoint)
+     payload.query_parameters←{0::⎕JSON'{}' ⋄ ⎕NS ⍵}⍵.QueryParams
+     ⍵.Response.Payload←payload
      0⊣⍵.Fail 404
  }

APLSource/AUTH/Reject.aplf

@@ -0,0 +1,15 @@
+ r←Reject req;json;payload
+⍝ Build the 401 envelope for a request that failed authentication.
+⍝ Jarvis converts a payload to JSON only for requests that reach a handler
+⍝ so this payload is serialised here and sent verbatim
+⍝ JSON sanitisation copied from Jarvis
+⍝ req: Jarvis request object    :: namespace reference
+⍝   r: Jarvis auth result       :: 1, meaning unauthorized
+ payload←(error:401 ⋄ message:'Missing or invalid X-API-Key header')
+ payload.request←(method:req.Method ⋄ endpoint:req.Endpoint)
+⍝ Pre-routing, so QueryParams is still Jarvis's name/value matrix, not Stark's namespace
+ payload.query_parameters←{0::⎕JSON'{}' ⋄ ()⎕VSET↓⍵}req.QueryParams
+ json←∊{127<c←⎕UCS ⍵:'\u','0123456789ABCDEF'[1+16 16 16 16⊤c] ⋄ ⍵}¨1 ⎕JSON payload
+ 'content-type'req.SetHeader'application/json'
+ req.Response.Payload←⎕UCS json
+ r←1

APLSource/AUTH/VerifyToken.aplf

@@ -2,11 +2,10 @@
 ⍝ ⍵: Jarvis HTTP request object
 ⍝ ←: Jarvis authentication result (0 for success, non-zero to fail request)
      h←⍵.Headers
-     r←⍵.Response
      i←(⊣/h)⍳⊂'x-api-key'
-     i>≢h:1      ⍝ FAIL 'No x-api-key header found'
+     i>≢h:Reject ⍵      ⍝ FAIL 'No x-api-key header found'
      try←h⊃⍨⊂2,⍨i
      key←##.GLOBAL.secrets.upload_token
-     try≢key:1   ⍝ FAIL 'Invalid x-api-key'
+     try≢key:Reject ⍵   ⍝ FAIL 'Invalid x-api-key'
      0
  }

APLSource/CACHE/Build.aplf

@@ -1,9 +1,9 @@
- cache←Build opt;Stem;Tokenise;dt_fields;ep;mat_fields;norm;pid;pname;search_columns;search_fields;stemmer;title;vids;weights;ytid
+ cache←Build opt;CountTerms;Stem;dt_fields;ep;mat_fields;norm;pid;pname;search_fields;stemmer;term_count;title;vids;weights;ytid
  ;GetEnv;ReadJSON;Unidecode;SQL;Table2JSON;TimeStamp2DDN
 ⍝ The external dependencies above are localised so that Setup's assignments appear as
 ⍝ Build's locals and vanish on return, otherwise certain names (e.g. .NET class instances)
 ⍝ prevent the CACHE namespace from being serialised into isolate processes.
-⍝ opt: options namespace (currently unused)
+⍝ opt: using and app_dir, computed in the main process
 ⍝ cache: cache namespace for wholesale replacement
 ⍝ Store pre-joined tables for API endpoints and query results
 ⍝ Create character and numeric matrices for filters
@@ -27,13 +27,14 @@
 ⍝ stored in APL as numeric 0 and 1 for efficient filtering
  :If 0=≢cache.events.data
      cache.events.has_videos←⍬
-     cache.events.data←''
+     cache.events.data←⍬
  :Else
      cache.events.has_videos←cache.events.data.has_videos
      cache.events.data.has_videos←⊂¨'false' 'true'[1+cache.events.has_videos]   ⍝ Convert APL Bool to JSON Bool
  :EndIf
 
  vids←cache.videos←⎕NS''   ⍝ Namespaces pass by reference, so assignment to vids shorthand updates cache namespace
+
  vids.fields←ep.videos.fields
  vids.values←SQL.Do ep.videos.sql
  (title ytid pname pid)←↓⍉SQL.Do ep.videos.presenter.sql   ⍝ YouTube ID, presenter name
@@ -70,16 +71,30 @@
      stemmed←stemmer.Stem⊆⍵
      stemmed.Value
  }
- Tokenise←{
-     text←('  '∘⍷~⍛/⊢)(⎕UCS 10 13)~⍨,⍕⍵
-     words←(⎕C ⎕A)(∊⍨⊆⊢)text
-     non_words←(' ',⎕C ⎕A)(~⍤∊⍨⊆⊢)text
-     ↑non_words,(⊢,Stem¨)words
+
+ CountTerms←{
+     text←{('  '∘⍷~⍛/⊢)(⎕UCS 10 13)~⍨,⍕⍵}¨⍵
+     words←(⎕C ⎕A)∘(∊⍨⊆⊢)¨text
+     non_words←(' ',⎕C ⎕A)∘(~⍤∊⍨⊆⊢)¨text
+     w_index←∪⊃,/,words
+     find_words←w_index∘⍳¨words
+     w_count←¯1+↑{{≢⍵}⌸(⍳≢w_index)⍪⍵}¨find_words
+   ⍝ Only stem the unique index words to reduce number of calls to .NET. Stemmer package
+   ⍝ FIXME modify package to accept a list of words, else implement in Dyalog
+     new←w_index≢¨stems←Stem¨w_index
+     w_count,←new/w_count   ⍝ Count of stemmed words is count of original words
+     w_index,←new/stems
+    ⍝ Add non_word term counts
+     nw_index←∪⊃,/,non_words
+     find_nw←nw_index∘⍳¨non_words
+     nw_count←¯1+↑{{≢⍵}⌸(⍳≢nw_index)⍪⍵}¨find_nw
+
+     index←↑w_index,nw_index
+     term_count←w_count,nw_count
+     index term_count
  }
- search_columns←Tokenise¨(vids.fields∊search_fields)/vids.index_all
- vids.index_width←0⌈⌈/(⊢/⍴)¨,search_columns
- search_columns←{(≢⍵)(vids.index_width)↑⍵}¨search_columns
- vids.index←∪⊃⍪/,search_columns
 
- vids.search←vids.index ComputeSearch weights search_columns
+ (vids.index term_count)←CountTerms(vids.fields∊search_fields)/vids.index_all
+
+ vids.search←vids.index ComputeSearch weights term_count
  vids.recommendations←ComputeRecommendations vids.search