perf(stats): serve 5chan stats on their own and keep threads only for popular boards

stats/monitor/README.md

@@ -29,7 +29,7 @@ The API listens on port `3000` by default:
 - `/history` returns stored snapshots over time
 - `/metrics/prometheus` exposes Prometheus metrics
 
-While it monitors community IPNS records, the monitor also writes every 5chan board's record time, stats and most popular threads to `exports/5chan-boards.json` once a minute, which the stats dashboard server serves at `/api/v1/5chan/boards` (see [`../web/README.md`](../web/README.md#5chan-boards-provider)).
+While it monitors community IPNS records, the monitor also writes every 5chan board's record time, stats and most popular threads to `exports/5chan-boards.json` once a minute, which the stats dashboard server serves at `/api/v1/5chan/boards` and `/api/v1/5chan/stats` (see [`../web/README.md`](../web/README.md#5chan-boards-provider)).
 
 ### Service probes and alerts
 

stats/monitor/config.js

@@ -20,7 +20,8 @@ const parseCsvEnv = (value) =>
     .filter(Boolean);
 
 // The monitor writes the 5chan boards snapshot here and the stats web server serves it at
-// /api/v1/5chan/boards. Compose mounts one volume at this directory in both containers.
+// /api/v1/5chan/boards and /api/v1/5chan/stats. Compose mounts one volume at this directory in both
+// containers.
 const fiveChanBoardsPath =
   process.env.BITSOCIAL_STATS_5CHAN_BOARDS_PATH ||
   path.join(monitorRoot, "exports", "5chan-boards.json");

stats/monitor/lib/client-sources.js

@@ -43,6 +43,11 @@ export const normalizeDirectory = async (
     publicKey,
     title: directory.title,
     directoryCode: directory.directoryCode,
+    // Undefined when the directory does not say; 5chan treats such a board as worksafe.
+    safeForWork:
+      typeof directory.features?.safeForWork === "boolean"
+        ? directory.features.safeForWork
+        : undefined,
     clientId,
     resolutionError,
   };

stats/monitor/lib/client-sources.test.js

@@ -21,12 +21,31 @@ test("preserves both the community name and public key", async () => {
       publicKey: "12D3KooWSpKszPM2c17KBgbnoRrkWPCHJosGGFg3bKnzarYhHeSc",
       title: "/an/ - Animals & Nature",
       directoryCode: "an",
+      safeForWork: undefined,
       clientId: "5chan",
       resolutionError: undefined,
     },
   );
 });
 
+test("keeps whether the directory marks the community worksafe", async () => {
+  const resolve = async () => "12D3KooWBoard";
+  for (const safeForWork of [true, false]) {
+    const community = await normalizeDirectory(
+      "5chan",
+      { name: "random-nsfw.bso", features: { safeForWork, bumpLimit: 300 } },
+      resolve,
+    );
+    assert.equal(community.safeForWork, safeForWork);
+  }
+  const undeclared = await normalizeDirectory(
+    "5chan",
+    { name: "random-nsfw.bso", features: { safeForWork: "false" } },
+    resolve,
+  );
+  assert.equal(undeclared.safeForWork, undefined);
+});
+
 test("keeps monitoring by community public key when BSO resolution fails", async () => {
   const community = await normalizeDirectory(
     "5chan",

stats/monitor/lib/five-chan-boards.js

@@ -3,8 +3,8 @@ import path from "node:path";
 
 // Keeps what the 5chan home page needs from every 5chan board the monitor fetches (its record's
 // updatedAt, its stats and its most popular thread) and writes it to a JSON file that the stats
-// web server serves at /api/v1/5chan/boards, so the home page does not have to download every
-// board over P2P before it can show its Popular Threads and Stats boxes.
+// web server serves at /api/v1/5chan/boards and /api/v1/5chan/stats, so the home page does not
+// have to download every board over P2P before it can show its Popular Threads and Stats boxes.
 
 export const FIVE_CHAN_CLIENT_ID = "5chan";
 export const FIVE_CHAN_BOARDS_VERSION = 1;
@@ -129,12 +129,13 @@ export const createFiveChanBoards = ({
       generatedAt: nowSeconds,
       boards: communities
         .filter(({ clientId }) => clientId === FIVE_CHAN_CLIENT_ID)
-        .map(({ address, publicKey, directoryCode }) => {
+        .map(({ address, publicKey, directoryCode, safeForWork }) => {
           const board = boards.get(address);
           return {
             address,
             publicKey: publicKey ?? null,
             directoryCode: directoryCode ?? null,
+            safeForWork: safeForWork ?? null,
             updatedAt: board?.updatedAt ?? null,
             stats: board?.stats ?? null,
             threads: board ? pickPopularThreads(board.candidates, nowSeconds) : [],

stats/monitor/lib/five-chan-boards.test.js

@@ -64,6 +64,7 @@ const board = (address, fields = {}) => ({
   address,
   publicKey: `12D3KooW${address}`,
   directoryCode: address.slice(0, 1),
+  safeForWork: true,
   clientId: "5chan",
   ...fields,
 });
@@ -234,7 +235,11 @@ test("lists every 5chan board, including boards never fetched", () => {
   fiveChanBoards.recordCommunityUpdate(seedit, communityRecord([pageEntry()]));
 
   assert.deepEqual(
-    fiveChanBoards.document([fetched, board("biz.bso", { publicKey: undefined }), seedit]),
+    fiveChanBoards.document([
+      fetched,
+      board("biz.bso", { publicKey: undefined, safeForWork: undefined }),
+      seedit,
+    ]),
     {
       version: 1,
       generatedAt: NOW,
@@ -243,6 +248,7 @@ test("lists every 5chan board, including boards never fetched", () => {
           address: "anime-and-manga.bso",
           publicKey: "12D3KooWanime-and-manga.bso",
           directoryCode: "a",
+          safeForWork: true,
           updatedAt: NOW - 120,
           stats: {
             statsCid: "QmStats",
@@ -257,6 +263,7 @@ test("lists every 5chan board, including boards never fetched", () => {
           address: "biz.bso",
           publicKey: null,
           directoryCode: "b",
+          safeForWork: null,
           updatedAt: null,
           stats: null,
           threads: [],

stats/web/README.md

@@ -15,14 +15,15 @@ The public dashboard at `https://stats.bitsocial.net/`: a Network overview at `/
 | `GET /api/v1/clients/:client?range=`                      | One client's totals, activity series and every community with its availability history                       |
 | `GET /api/v1/clients/:client/communities/:address?range=` | One community's activity, peer and availability series                                                       |
 | `GET /api/v1/5chan/boards`                                | Every 5chan board's record time, stats and most popular threads, for the 5chan home page (see below)         |
+| `GET /api/v1/5chan/stats`                                 | The same boards without their threads, for the 5chan home page's Stats box                                   |
 
 Ranges are `6h`, `24h` (default), `7d` and `14d`. Series share the response's `range.start` and `range.step`; availability values are the share of each step a check passed.
 
 The old Grafana shared-dashboard URLs (`/public-dashboards/<token>`) redirect to the matching view.
 
 ### 5chan boards provider
 
-`GET /api/v1/5chan/boards` lets the 5chan home page render its Popular Threads and Stats boxes at once instead of first downloading every board over P2P. It lists every board in every active 5chan directory, including the boards competing with each directory's default, and takes no parameters (any query string is ignored).
+`GET /api/v1/5chan/boards` lets the 5chan home page render its Popular Threads and Stats boxes at once instead of first downloading every board over P2P. It lists every board in every active 5chan directory, including the boards competing with each directory's default, and takes no parameters (any query string is ignored). `GET /api/v1/5chan/stats` is the same document with every board's `threads` left out, so the Stats box waits for about 8 KB instead of the whole boards document.
 
 ```json
 {
@@ -33,6 +34,7 @@ The old Grafana shared-dashboard URLs (`/public-dashboards/<token>`) redirect to
       "address": "anime-and-manga.bso",
       "publicKey": "12D3KooW...",
       "directoryCode": "a",
+      "safeForWork": true,
       "updatedAt": 1792499000,
       "stats": { "statsCid": "Qm...", "allPostCount": 123, "allReplyCount": 456, "weekActiveUserCount": 7 },
       "threads": [{ "cid": "Qm...", "raw": { "comment": {}, "commentUpdate": {} } }]
@@ -43,11 +45,12 @@ The old Grafana shared-dashboard URLs (`/public-dashboards/<token>`) redirect to
 
 - `generatedAt` (when the monitor last picked the threads) and `updatedAt` (the board record's own `updatedAt`) are unix seconds. A board the monitor has not fetched yet has `updatedAt: null`, `stats: null` and `threads: []`.
 - `stats` holds the count fields of the stats file the record's `statsCid` points to (`allPostCount`, `allReplyCount`, `weekActiveUserCount` and the other hour to all-time `ActiveUser`/`Post`/`Reply` counts) plus that `statsCid`.
-- `threads` holds at most one thread: the board's most popular one from the page its record preloads (the first key of `posts.pages`, which for 5chan boards is `active`, not `hot`; a record with no preloaded page has none). Pinned, locked, removed and deleted threads and threads without a link are skipped, the rest are ranked by the 5chan home page's score, `max(replyCount, 0.1) / (1 + age / 72h)` with the age counted from `lastReplyTimestamp`, else the thread's `timestamp`, and the best thread with media (a `thumbnailUrl`, or a link to an image, video or audio file) wins over a better-scored plain link. A board with only plain links still gets its best one.
-- `raw.comment` and `raw.commentUpdate` are the objects exactly as the board published them. That includes `commentUpdate.replies`, the signed preview of the thread's replies, which makes up most of the payload and cannot be left out without breaking the signature. With the real 68 boards in October 2026 the response was about 0.8 MB, 0.17 MB gzipped.
+- `safeForWork` is the directory's `features.safeForWork` for the board, or `null` when it does not say; 5chan treats only `false` as NSFW.
+- `threads` holds at most one thread: the board's most popular one from the page its record preloads (the first key of `posts.pages`, which for 5chan boards is `active`, not `hot`; a record with no preloaded page has none). Pinned, locked, removed and deleted threads and threads without a link are skipped, the rest are ranked by the 5chan home page's score, `max(replyCount, 0.1) / (1 + age / 72h)` with the age counted from `lastReplyTimestamp`, else the thread's `timestamp`, and the best thread with media (a `thumbnailUrl`, or a link to an image, video or audio file) wins over a better-scored plain link. A board with only plain links still gets its best one. Only the 20 worksafe boards and the 20 NSFW boards with the best-scoring threads keep theirs; every other board has `threads: []`. The home page picks from the threads of the 16 most popular boards it shows, and its visitors show worksafe boards, NSFW boards or both, so each group's top 20 covers every choice with room for boards it skips.
+- `raw.comment` and `raw.commentUpdate` are the objects exactly as the board published them. That includes `commentUpdate.replies`, the signed preview of the thread's replies, which makes up most of the payload and cannot be left out without breaking the signature. With the real 68 boards in October 2026 the response was about 0.8 MB, 0.17 MB gzipped, before it kept only the most popular boards' threads; with them it was 0.12 MB gzipped.
 - Clients must treat the threads like any other page from a peer: verify the author signature on `raw.comment`, the community signature on `raw.commentUpdate`, and that `raw.comment` hashes to `cid`. The stats numbers cannot be verified that way; they are what this provider read from each board's `statsCid`.
 
-How it updates: on each IPNS pass (every 10 minutes) the monitor keeps every 5chan board's record and stats, and once a minute writes the boards to `stats/monitor/exports/5chan-boards.json` (`BITSOCIAL_STATS_5CHAN_BOARDS_PATH` overrides the path), replacing it atomically. Compose shares that directory between the monitor and this server through a volume (`stats-monitor-exports` locally, `bitsocial-stats-monitor-exports` in [`../deploy/compose.yaml`](../deploy/compose.yaml)). This server reads the file on its own 60 second schedule, never per request, and answers from memory with an ETag, `Cache-Control: public, max-age=60` and `Access-Control-Allow-Origin: *`, since the data is public and read from 5chan's web, Android and Electron origins. It also sends `Cloudflare-CDN-Cache-Control: max-age=60, stale-while-revalidate=600`, which only Cloudflare reads and does not forward: once its copy expires, Cloudflare keeps serving it for up to 10 minutes while it refetches in the background, so visitors do not wait for this server. Like the other API documents, the body is gzipped once per snapshot and sent compressed to clients that accept gzip; Caddy's `encode gzip` passes an already-encoded response through, so the Caddy block needs no change. A restarted monitor starts from its last file, and a file that fails to parse keeps the previous snapshot. Until the first file exists the endpoint answers `502`.
+How it updates: on each IPNS pass (every 10 minutes) the monitor keeps every 5chan board's record and stats, and once a minute writes the boards to `stats/monitor/exports/5chan-boards.json` (`BITSOCIAL_STATS_5CHAN_BOARDS_PATH` overrides the path), replacing it atomically. Compose shares that directory between the monitor and this server through a volume (`stats-monitor-exports` locally, `bitsocial-stats-monitor-exports` in [`../deploy/compose.yaml`](../deploy/compose.yaml)). This server reads the file on its own 60 second schedule, never per request, and answers both endpoints from memory with an ETag, `Cache-Control: public, max-age=60` and `Access-Control-Allow-Origin: *`, since the data is public and read from 5chan's web, Android and Electron origins. It also sends `Cloudflare-CDN-Cache-Control: max-age=60, stale-while-revalidate=600`, which only Cloudflare reads and does not forward: once its copy expires, Cloudflare keeps serving it for up to 10 minutes while it refetches in the background, so visitors do not wait for this server. Like the other API documents, the body is gzipped once per snapshot and sent compressed to clients that accept gzip; Caddy's `encode gzip` passes an already-encoded response through, so the Caddy block needs no change. A restarted monitor starts from its last file, and a file that fails to parse keeps the previous snapshot. Until the first file exists the endpoint answers `502`.
 
 ### Development
 

stats/web/server/five-chan-boards.mjs

@@ -1,15 +1,52 @@
 import fs from "node:fs/promises";
-import { FIVE_CHAN_BOARDS_VERSION } from "../../monitor/lib/five-chan-boards.js";
+import {
+  FIVE_CHAN_BOARDS_VERSION,
+  isPopularThreadCandidate,
+  popularityScore,
+} from "../../monitor/lib/five-chan-boards.js";
 
-// The document behind GET /api/v1/5chan/boards: each 5chan board's record time, stats and most
-// popular threads, which the monitor writes to a file (stats/monitor/lib/five-chan-boards.js).
-// The snapshot store reads that file on its own schedule, so requests never touch the file or
-// cause any upstream work.
+// The documents behind GET /api/v1/5chan/boards and GET /api/v1/5chan/stats: each 5chan board's
+// record time, stats and, for the boards the home page can pick from, most popular thread, which
+// the monitor writes to a file (stats/monitor/lib/five-chan-boards.js). The snapshot store reads
+// that file on its own schedule, so requests never touch the file or cause any upstream work.
 
 export const FIVE_CHAN_BOARDS_KEY = "5chan:boards";
+export const FIVE_CHAN_STATS_KEY = "5chan:stats";
 // How often the snapshot is rebuilt, and how long clients may cache it. The monitor rewrites the
 // file every minute too.
 export const FIVE_CHAN_BOARDS_SECONDS = 60;
+// The 5chan home page fills its Popular Threads box from the threads of the 16 most popular boards
+// it shows (CANDIDATE_POOL_SIZE in 5chan's src/lib/popular-threads.ts), and its visitors show
+// worksafe boards, NSFW boards or both. The most popular boards of each group cover all three
+// choices; the 4 past 16 stand in for boards 5chan skips, such as one whose thread shows a picture
+// a more popular board already shows.
+const POPULAR_BOARDS_PER_GROUP = 20;
+
+// The score of a board's best thread, or undefined for a board without one.
+const boardScore = (board, nowSeconds) => {
+  const scores = (Array.isArray(board.threads) ? board.threads : [])
+    .map((thread) => thread?.raw)
+    .filter(isPopularThreadCandidate)
+    .map((raw) => popularityScore(raw, nowSeconds));
+  return scores.length ? Math.max(...scores) : undefined;
+};
+
+// Every other board keeps its stats but sends no thread: each thread carries the signed preview
+// of its replies, which makes up most of the document. 5chan reads a board as NSFW only when its
+// directory says it is not worksafe.
+export const keepPopularBoardThreads = (boards, nowSeconds) => {
+  const popularBoards = new Set();
+  for (const nsfw of [false, true]) {
+    boards
+      .filter((board) => (board.safeForWork === false) === nsfw)
+      .map((board) => ({ board, score: boardScore(board, nowSeconds) }))
+      .filter(({ score }) => score !== undefined)
+      .sort((left, right) => right.score - left.score)
+      .slice(0, POPULAR_BOARDS_PER_GROUP)
+      .forEach(({ board }) => popularBoards.add(board));
+  }
+  return boards.map((board) => (popularBoards.has(board) ? board : { ...board, threads: [] }));
+};
 
 export const createFiveChanBoardsSource = ({ filePath, readFile = fs.readFile }) => {
   let lastError;
@@ -27,7 +64,21 @@ export const createFiveChanBoardsSource = ({ filePath, readFile = fs.readFile })
         }
         lastError = undefined;
         return {
-          documents: new Map([[FIVE_CHAN_BOARDS_KEY, { version, generatedAt, boards }]]),
+          documents: new Map([
+            [
+              FIVE_CHAN_BOARDS_KEY,
+              { version, generatedAt, boards: keepPopularBoardThreads(boards, generatedAt) },
+            ],
+            // The Stats box needs a few numbers per board, so it does not wait for any thread.
+            [
+              FIVE_CHAN_STATS_KEY,
+              {
+                version,
+                generatedAt,
+                boards: boards.map(({ threads: _threads, ...board }) => board),
+              },
+            ],
+          ]),
           errors: [],
         };
       } catch (error) {