feat(news): completeness measurement — feed health w/ silent zeros, coverage ledger + provenance, GDELT recall benchmark (#4920)

.github/workflows/feed-validation.yml

@@ -24,7 +24,10 @@ on:
     branches: [main]
     paths:
       - 'src/config/feeds.ts'
+      - 'server/worldmonitor/news/v1/_feeds.ts'
       - 'scripts/validate-rss-feeds.mjs'
+      - 'scripts/_feed-health.mjs'
+      - 'scripts/seed-recall-benchmark.mjs'
       - 'api/_rss-allowed-domain-match.js'
       - 'api/_rss-allowed-domains.js'
       - 'shared/rss-allowed-domains.json'
@@ -43,12 +46,35 @@ jobs:
     # nominal, but a hung dependency-install or network stall could otherwise
     # occupy a runner for the default 6h timeout. 15min gives plenty of
     # headroom while avoiding stuck-runner alerts on real outages.
-    timeout-minutes: 15
+    timeout-minutes: 25
     steps:
       - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
         with:
           node-version: '24'
           cache: 'npm'
       - run: npm ci
+      # #4920: with Upstash secrets configured, the run also publishes
+      # per-feed health + silent-zero streaks (news:feed-health:v1) and the
+      # GDELT recall benchmark (news:recall-benchmark:v1). Without secrets
+      # both publishers skip silently — validation output is unchanged.
+      # #4927 review P1: a silent skip must still be VISIBLE on the run —
+      # otherwise green scheduled runs mask never-activated health
+      # publishing (the health endpoints activation-gate these keys, but an
+      # operator scanning workflow runs should see the gap too).
+      - name: Warn when health publishing is not activated
+        env:
+          UPSTASH_REDIS_REST_URL: ${{ secrets.UPSTASH_REDIS_REST_URL }}
+          UPSTASH_REDIS_REST_TOKEN: ${{ secrets.UPSTASH_REDIS_REST_TOKEN }}
+        run: |
+          if [ -z "$UPSTASH_REDIS_REST_URL" ] || [ -z "$UPSTASH_REDIS_REST_TOKEN" ]; then
+            echo "::warning title=feed-health publishing skipped::UPSTASH_REDIS_REST_URL/TOKEN are not configured as repo secrets — news:feed-health:v1 and news:recall-benchmark:v1 will not be published (health endpoints show these as pending-activation)."
+          fi
       - run: npm run test:feeds:ci
+        env:
+          UPSTASH_REDIS_REST_URL: ${{ secrets.UPSTASH_REDIS_REST_URL }}
+          UPSTASH_REDIS_REST_TOKEN: ${{ secrets.UPSTASH_REDIS_REST_TOKEN }}
+      - run: node scripts/seed-recall-benchmark.mjs
+        env:
+          UPSTASH_REDIS_REST_URL: ${{ secrets.UPSTASH_REDIS_REST_URL }}
+          UPSTASH_REDIS_REST_TOKEN: ${{ secrets.UPSTASH_REDIS_REST_TOKEN }}

api/health.js

@@ -108,6 +108,10 @@ const BOOTSTRAP_KEYS = {
 };
 
 const STANDALONE_KEYS = {
+  // #4920 completeness measurement (daily GH Actions publishers) — ops
+  // keys: health-monitored but NOT bootstrap-hydrated into page loads.
+  newsFeedHealth:    'news:feed-health:v1',
+  newsRecallBenchmark: 'news:recall-benchmark:v1',
   serviceStatuses:       'infra:service-statuses:v1',
   macroSignals:          'economic:macro-signals:v1',
   bisPolicy:             'economic:bis:policy:v1',
@@ -260,6 +264,9 @@ const SEED_META = {
   notamClosures:    { key: 'seed-meta:aviation:notam',          maxStaleMin: 240 }, // 2h interval; 240min = 2x interval
   predictionMarkets: { key: 'seed-meta:prediction:markets',     maxStaleMin: 90 },
   newsInsights:     { key: 'seed-meta:news:insights',           maxStaleMin: 30 },
+  // #4920: daily GH Actions cadence; 2880 = 2x — one fully missed day alarms
+  newsFeedHealth:   { key: 'seed-meta:news:feed-health',        maxStaleMin: 2880 },
+  newsRecallBenchmark: { key: 'seed-meta:news:recall-benchmark', maxStaleMin: 2880 },
   marketQuotes:     { key: 'seed-meta:market:stocks',         maxStaleMin: 30 },
   commodityQuotes:  { key: 'seed-meta:market:commodities',    maxStaleMin: 30 },
   goldExtended:     { key: 'seed-meta:market:gold-extended',  maxStaleMin: 30 },
@@ -486,6 +493,14 @@ const ON_DEMAND_KEYS = new Set([
   // chronic LLM-provider failures must surface as CRIT.
   'simulationPackageLatest', // written by writeSimulationPackage after deep forecast runs; only present after first successful deep run
   'simulationOutcomeLatest', // written by writeSimulationOutcome after simulation runs; only present after first successful simulation
+  // #4927 review P1: activation-gated on the operator adding UPSTASH_* as GH
+  // Actions secrets — the publishers skip silently without them, so a
+  // never-activated key must read as soft EMPTY_ON_DEMAND, not EMPTY/CRIT,
+  // while the workflow stays green. On-demand softening is REVOKED once the
+  // durable activation marker exists (see ACTIVATION_MARKERS): after the
+  // first publish, missing data/meta is EMPTY/STALE_SEED like any other key.
+  'newsFeedHealth',
+  'newsRecallBenchmark',
   'newsThreatSummary', // relay classify loop — only written when mergedByCountry has entries; absent on quiet news periods
   'resilienceRanking', // on-demand RPC cache populated after ranking requests; missing before first Pro use is expected
   'recoveryFiscalSpace', 'recoveryReserveAdequacy', 'recoveryExternalDebt',
@@ -537,6 +552,17 @@ const ON_DEMAND_KEYS = new Set([
 // even when the data key has strlen=0. For producer-specific cases where the
 // payload must exist but recordCount=0 is valid, add to ZERO_RECORD_DATA_OK_KEYS
 // only.
+// #4927 re-review P1: "has ever published" must survive the 7d seed-meta
+// TTL — inferring activation from meta existence meant a publisher that ran
+// once and died read as healthy pending-activation again after the meta
+// expired. Publishers SET these markers with NO TTL on every successful
+// publish; when the marker exists the key leaves ON_DEMAND softening and
+// normal EMPTY/STALE_SEED rules apply.
+const ACTIVATION_MARKERS = {
+  newsFeedHealth: 'seed-activated:news:feed-health',
+  newsRecallBenchmark: 'seed-activated:news:recall-benchmark',
+};
+
 const EMPTY_DATA_OK_KEYS = new Set([
   'notamClosures', 'faaDelays', 'intlDelays', 'gpsjam', 'positiveGeoEvents', 'weatherAlerts',
   'earningsCalendar', 'econCalendar', 'cotPositioning',
@@ -675,7 +701,8 @@ function isCascadeCovered(name, hasData, keyStrens, keyErrors) {
 function classifyKey(name, redisKey, opts, ctx) {
   const { keyStrens, keyErrors, keyMetaValues, keyMetaErrors, now } = ctx;
   const seedCfg = SEED_META[name];
-  const isOnDemand = !!opts.allowOnDemand && ON_DEMAND_KEYS.has(name);
+  const isOnDemand = !!opts.allowOnDemand && ON_DEMAND_KEYS.has(name)
+    && !(ctx.activatedNames && ctx.activatedNames.has(name));
 
   const meta = readSeedMeta(seedCfg, keyMetaValues, keyMetaErrors, now);
 
@@ -850,6 +877,7 @@ export default async function handler(req, ctx) {
     ...Object.values(STANDALONE_KEYS),
   ];
   const allMetaKeys = Object.values(SEED_META).map(s => s.key);
+  const activationEntries = Object.entries(ACTIVATION_MARKERS);
 
   // STRLEN for data keys avoids loading large blobs into memory (OOM prevention).
   // NEG_SENTINEL ('__WM_NEG__') is 10 bytes — strlenIsData() rejects exactly
@@ -859,6 +887,7 @@ export default async function handler(req, ctx) {
     const commands = [
       ...allDataKeys.map(k => ['STRLEN', k]),
       ...allMetaKeys.map(k => ['GET', k]),
+      ...activationEntries.map(([, marker]) => ['EXISTS', marker]),
     ];
     if (!getRedisCredentials()) throw new Error('Redis not configured');
     results = await redisPipeline(commands, 8_000);
@@ -895,8 +924,16 @@ export default async function handler(req, ctx) {
     if (r?.error) keyMetaErrors.set(allMetaKeys[i], r.error);
     keyMetaValues.set(allMetaKeys[i], r?.result ?? null);
   }
+  // activatedNames: keys whose durable activation marker exists — these
+  // leave ON_DEMAND softening permanently (#4927 re-review P1). A marker
+  // read error degrades to not-activated (soft), never to a false alarm.
+  const activatedNames = new Set();
+  for (let i = 0; i < activationEntries.length; i++) {
+    const r = results[allDataKeys.length + allMetaKeys.length + i];
+    if (!r?.error && Number(r?.result) === 1) activatedNames.add(activationEntries[i][0]);
+  }
 
-  const classifyCtx = { keyStrens, keyErrors, keyMetaValues, keyMetaErrors, now };
+  const classifyCtx = { keyStrens, keyErrors, keyMetaValues, keyMetaErrors, activatedNames, now };
   const checks = {};
   const counts = { ok: 0, warn: 0, onDemandWarn: 0, staleContent: 0, crit: 0 };
   let totalChecks = 0;
@@ -1042,6 +1079,7 @@ export default async function handler(req, ctx) {
 export const __testing__ = {
   readSeedMeta,
   classifyKey,
+  ACTIVATION_MARKERS,
   STATUS_COUNTS,
   // U7 (Tier 3 parity test): exposed for tests/mcp-bootstrap-parity.test.mjs
   // to walk the canonical seeded-data inventory. Both consts are unexported

api/seed-health.js

@@ -27,6 +27,17 @@ const SEED_DOMAINS = {
   'climate:co2-monitoring':   { key: 'seed-meta:climate:co2-monitoring',   intervalMin: 1440 }, // daily cron; health.js maxStaleMin:4320 (3x) is intentionally higher — it's an alarm threshold, not the cron cadence
   'climate:ocean-ice':        { key: 'seed-meta:climate:ocean-ice',        intervalMin: 1440 }, // daily cron; health.js maxStaleMin:2880 (2x) tolerates one missed run
   'climate:news-intelligence': { key: 'seed-meta:climate:news-intelligence', intervalMin: 30 },
+  // #4920 completeness measurement — both run in the daily feed-validation
+  // GitHub Actions workflow (00:00 UTC), not Railway. 1440-min cadence;
+  // classifier stales at intervalMin*2 = one fully missed day.
+  // activationKey (#4927 review P1 + re-review): published from GH Actions
+  // only when the operator has added the UPSTASH secrets. 'missing' reads
+  // as pending-activation (healthy) ONLY while the durable activation
+  // marker is absent — publishers SET it with no TTL on first success, so
+  // "has ever published" survives the 7d seed-meta TTL and a dead
+  // publisher alarms as missing/stale instead of reverting to pending.
+  'news:feed-health':         { key: 'seed-meta:news:feed-health',         intervalMin: 1440, activationKey: 'seed-activated:news:feed-health' },
+  'news:recall-benchmark':    { key: 'seed-meta:news:recall-benchmark',    intervalMin: 1440, activationKey: 'seed-activated:news:recall-benchmark' },
   // Phase 2 — Parameterized endpoints
   'unrest:events':            { key: 'seed-meta:unrest:events',            intervalMin: 15 },
   'cyber:threats':            { key: 'seed-meta:cyber:threats',            intervalMin: 240 },
@@ -262,13 +273,18 @@ async function getSeedBatch(entries) {
   const commands = [];
   const metaSlots = [];
   const probeSlots = [];
+  const activationSlots = [];
   for (const [domain, cfg] of entries) {
     metaSlots.push({ domain, key: cfg.key, index: commands.length });
     commands.push(['GET', cfg.key]);
     if (cfg.dataProbe?.key) {
       probeSlots.push({ domain, index: commands.length });
       commands.push(['GET', cfg.dataProbe.key]);
     }
+    if (cfg.activationKey) {
+      activationSlots.push({ domain, index: commands.length });
+      commands.push(['EXISTS', cfg.activationKey]);
+    }
   }
 
   const data = await redisPipeline(commands, 3000);
@@ -286,7 +302,11 @@ async function getSeedBatch(entries) {
   for (const slot of probeSlots) {
     probeMap.set(slot.domain, data[slot.index]?.result ?? null);
   }
-  return { metaMap, probeMap };
+  const activatedMap = new Map();
+  for (const slot of activationSlots) {
+    activatedMap.set(slot.domain, Number(data[slot.index]?.result) === 1);
+  }
+  return { metaMap, probeMap, activatedMap };
 }
 
 export default async function handler(req) {
@@ -305,9 +325,10 @@ export default async function handler(req) {
   const entries = Object.entries(SEED_DOMAINS);
 
   let metaMap;
+  let activatedMap = new Map();
   let probeMap;
   try {
-    ({ metaMap, probeMap } = await getSeedBatch(entries));
+    ({ metaMap, probeMap, activatedMap } = await getSeedBatch(entries));
   } catch {
     return jsonResponse({ error: 'Redis unavailable' }, 503, cors);
   }
@@ -321,6 +342,15 @@ export default async function handler(req) {
     const maxStalenessMs = cfg.intervalMin * 2 * 60 * 1000;
 
     if (!meta) {
+      if (cfg.activationKey && !activatedMap.get(domain)) {
+        // Never seeded (durable marker absent) AND operator-activation-
+        // gated: healthy pending state, not an alarm (#4927 review P1).
+        // Once the marker exists, missing meta falls through to
+        // 'missing' — a publisher that ran once and died must alarm
+        // (#4927 re-review P1).
+        seeds[domain] = { status: 'pending-activation', fetchedAt: null, recordCount: null, stale: false };
+        continue;
+      }
       seeds[domain] = { status: 'missing', fetchedAt: null, recordCount: null, stale: true };
       if (cfg.minRecordCount != null) seeds[domain].minRecordCount = cfg.minRecordCount;
       missingCount++;

docs/api/NewsService.openapi.yaml

@@ -739,7 +739,11 @@ components:
                 sourceCount:
                     type: integer
                     format: int32
-                    description: Number of unique sources that reported this story (cached from Redis Set).
+                    description: |-
+                        Number of unique sources reporting this story in the CURRENT digest
+                         cycle (batch corroboration from fuzzy story clustering, #4919;
+                         defaults to 1 for unclustered items). The cross-cycle Redis source
+                         set (story:sources:v1) feeds importance scoring, not this field.
                 phase:
                     type: string
                     enum:

docs/api/worldmonitor.openapi.yaml

@@ -32212,7 +32212,11 @@ components:
                 sourceCount:
                     type: integer
                     format: int32
-                    description: Number of unique sources that reported this story (cached from Redis Set).
+                    description: |-
+                        Number of unique sources reporting this story in the CURRENT digest
+                         cycle (batch corroboration from fuzzy story clustering, #4919;
+                         defaults to 1 for unclustered items). The cross-cycle Redis source
+                         set (story:sources:v1) feeds importance scoring, not this field.
                 phase:
                     type: string
                     enum:

proto/worldmonitor/news/v1/news_item.proto

@@ -58,7 +58,10 @@ message StoryMeta {
   int64 first_seen = 1 [(sebuf.http.int64_encoding) = INT64_ENCODING_NUMBER];
   // Total number of digest cycles in which this story appeared.
   int32 mention_count = 2;
-  // Number of unique sources that reported this story (cached from Redis Set).
+  // Number of unique sources reporting this story in the CURRENT digest
+  // cycle (batch corroboration from fuzzy story clustering, #4919;
+  // defaults to 1 for unclustered items). The cross-cycle Redis source
+  // set (story:sources:v1) feeds importance scoring, not this field.
   int32 source_count = 3;
   // Story lifecycle phase derived from persistence data.
   StoryPhase phase = 4;

scripts/_clustering.mjs

@@ -304,29 +304,60 @@ export function computeEntityCorroboration(clusters, nowMs = Date.now()) {
 
 // Note: velocity filter omitted (vs frontend selectTopStories) because digest
 // items lack velocity data. Phase B may add velocity when RPC provides it.
-export function selectTopStories(clusters, maxCount = 8) {
+/**
+ * @param {object[]} clusters
+ * @param {number} [maxCount]
+ * @param {{ considered?: number; admissibilityDropped?: number; sourceCapDropped?: number; overflowDropped?: number }} [stats]
+ *   #4920 coverage ledger: when provided, populated with how many clusters
+ *   each gate dropped — previously all three gates were silent.
+ */
+export function selectTopStories(clusters, maxCount = 8, stats) {
   const nowMs = Date.now();
   computeEntityCorroboration(clusters, nowMs);
-  const scored = clusters
-    .map(c => {
-      const score = scoreImportance(c);
-      return { cluster: c, score, effectiveScore: score * recencyWeight(c, nowMs) };
-    })
-    .filter(({ cluster: c, score }) => isTopStoriesAdmissible(c, score))
-    .sort((a, b) => b.effectiveScore - a.effectiveScore || b.score - a.score);
+  const admissible = [];
+  let admissibilityDropped = 0;
+  for (const c of clusters) {
+    const score = scoreImportance(c);
+    if (isTopStoriesAdmissible(c, score)) {
+      admissible.push({ cluster: c, score, effectiveScore: score * recencyWeight(c, nowMs) });
+    } else {
+      admissibilityDropped++;
+    }
+  }
+  admissible.sort((a, b) => b.effectiveScore - a.effectiveScore || b.score - a.score);
 
   const selected = [];
   const sourceCount = new Map();
   const MAX_PER_SOURCE = 3;
-
-  for (const { cluster, score, effectiveScore } of scored) {
+  let sourceCapDropped = 0;
+  let overflowDropped = 0;
+
+  // #4927 review P2: classify EVERY admissible candidate — breaking at
+  // maxCount lumped later same-source candidates into overflow arithmetic
+  // even though the source cap would have rejected them regardless of
+  // room. Cap-first attribution: a candidate whose source already hit the
+  // per-source cap is a sourceCap drop; only genuinely rankable candidates
+  // count as overflow.
+  for (const { cluster, score, effectiveScore } of admissible) {
     const source = cluster.primarySource;
     const count = sourceCount.get(source) || 0;
-    if (count < MAX_PER_SOURCE) {
-      selected.push({ ...cluster, importanceScore: score, effectiveImportanceScore: effectiveScore });
-      sourceCount.set(source, count + 1);
+    if (count >= MAX_PER_SOURCE) {
+      sourceCapDropped++;
+      continue;
     }
-    if (selected.length >= maxCount) break;
+    if (selected.length >= maxCount) {
+      overflowDropped++;
+      continue;
+    }
+    selected.push({ ...cluster, importanceScore: score, effectiveImportanceScore: effectiveScore });
+    sourceCount.set(source, count + 1);
+  }
+
+  if (stats && typeof stats === 'object') {
+    stats.considered = clusters.length;
+    stats.admissibilityDropped = admissibilityDropped;
+    stats.sourceCapDropped = sourceCapDropped;
+    stats.overflowDropped = overflowDropped;
   }
 
   return selected;