diff --git a/scripts/_insights-brief.mjs b/scripts/_insights-brief.mjs index 26e93428f1..64e225a962 100644 --- a/scripts/_insights-brief.mjs +++ b/scripts/_insights-brief.mjs @@ -2,6 +2,11 @@ // so tests can import without triggering the top-level runSeed() call. import { isBriefLeadEligible } from './_clustering.mjs'; +import { + validateNoHallucinatedProperNouns, + checkLeadGrounding, + verifyCitationIndexes, +} from './shared/brief-llm-core.js'; /** * Choose which clustered story to summarize for the WORLD BRIEF. @@ -43,3 +48,194 @@ Rules: export function briefUserPrompt(headline) { return `Headline: ${headline}\n\nRewrite as 2 sentences using only facts from this headline.`; } + +// ═══════════════════════════════════════════════════════════════════════════ +// #4921 — top-8 synthesis. The World Brief previously narrated ONE headline; +// these builders produce a genuine synthesis: a cited lead plus one line per +// top story, in a single structured LLM call. +// ═══════════════════════════════════════════════════════════════════════════ + +export function synthesisSystemPrompt(dateISO) { + return `Current date: ${dateISO}. + +You are compiling the WORLD BRIEF from the numbered stories below. Respond with JSON ONLY (no markdown fences, no commentary): +{"lead": "...", "lines": [{"n": 1, "text": "..."}, ...]} + +Rules: +- "lead": 2-3 sentences, under 80 words, synthesizing the most consequential 2-3 threads. Cite every claim with the bracket number of its story, e.g. [1] or [3]. +- "lines": exactly one entry per numbered story, in order. Each "text" is ONE sentence under 30 words restating that story, ending with its citation [n]. +- Use ONLY facts present in the numbered story text. Do not add names, places, dates, numbers, or context that are not explicitly there. +- Do not invent proper nouns (people, organizations, countries) that are not in the story text. +- Never merge facts from different stories into one claim; the lead may JUXTAPOSE stories but each claim keeps its own [n]. +- NEVER start with "Breaking news", "Good evening", "Tonight", or TV-style openings.`; +} + +export function synthesisUserPrompt(stories) { + const lines = stories.map((story, i) => { + const sources = Array.isArray(story.sources) && story.sources.length > 0 + ? story.sources.length + : (story.sourceCount ?? 1); + return `${i + 1}. ${story.primaryTitle} (${story.primarySource}, ${sources} source${sources === 1 ? '' : 's'})`; + }); + return `Stories:\n${lines.join('\n')}\n\nCompile the world brief JSON.`; +} + +/** + * Tolerant parser for the synthesis JSON. Strips code fences (groq and + * Gemini both wrap), extracts the outermost object, validates shape. + * Returns { lead, lines: [{ n, text }] } or null — callers fall back to + * the single-headline path on null (the brief always ships). + */ +export function parseBriefSynthesis(rawText, storyCount) { + if (typeof rawText !== 'string' || rawText.length === 0) return null; + const text = rawText.replace(/```(?:json)?/gi, '').trim(); + const start = text.indexOf('{'); + if (start === -1) return null; + // Balanced, string-aware brace scan (#4928 external review): a stray + // '}' in trailing prose defeated lastIndexOf-based slicing. + let end = -1; + let depth = 0; + let inString = false; + let escaped = false; + for (let i = start; i < text.length; i++) { + const ch = text[i]; + if (inString) { + if (escaped) escaped = false; + else if (ch === '\\') escaped = true; + else if (ch === '"') inString = false; + continue; + } + if (ch === '"') inString = true; + else if (ch === '{') depth++; + else if (ch === '}') { + depth--; + if (depth === 0) { end = i; break; } + } + } + if (end === -1) return null; + let parsed; + try { + parsed = JSON.parse(text.slice(start, end + 1)); + } catch { + return null; + } + const lead = typeof parsed?.lead === 'string' ? parsed.lead.trim() : ''; + if (lead.length < 40 || lead.length > 700) return null; + const rawLines = Array.isArray(parsed?.lines) ? parsed.lines : []; + const byIndex = new Map(); + for (const entry of rawLines) { + const n = Number(entry?.n); + const lineText = typeof entry?.text === 'string' ? entry.text.trim() : ''; + if (!Number.isInteger(n) || n < 1 || n > storyCount) continue; + if (lineText.length < 15 || lineText.length > 260) continue; + if (!byIndex.has(n)) byIndex.set(n, lineText); + } + // Require at least half the stories to have usable lines — below that + // the model ignored the contract and the single-headline fallback is + // more trustworthy. Missing lines are filled from headlines upstream. + if (byIndex.size < Math.ceil(storyCount / 2)) return null; + return { + lead, + lines: Array.from(byIndex.entries()) + .sort((a, b) => a[0] - b[0]) + .map(([n, lineText]) => ({ n, text: lineText })), + }; +} + +/** + * #4921/#4928: assemble the synthesized brief from a raw LLM response — + * pure and fully unit-testable. Applies the whole contract: + * - parse (fence-tolerant JSON, ≥half the stories lined) + * - editorial gate: at least one top story must be corroborated + * (≥2 sources / entity corroboration) — the synthesis path must not + * lower the legacy corroboration bar on all-single-source days + * - lead: proper-noun validation against ALL story titles (enforce → + * reject to fallback), anchor grounding, citation-index verification + * - lines: per-story proper-noun enforcement (a failing line degrades + * to its own headline, keeping its [n] so the citation contract holds) + * - sources: STRICT lockstep with citation indexes — entry i is always + * story i+1, substituting a minimal fallback when a story lacks a + * usable link (never filtered, or every later [n] would shift) + * + * @returns {null | { + * lead: string; + * lines: Array<{ n: number; text: string }>; + * sources: Array<{ title: string; source: string; url: string }>; + * hallucinatedLines: number; + * strippedCitations: number; + * }} null → caller falls back to the legacy single-headline path. + */ +export function composeSynthesizedBrief(rawText, topStories, opts = {}) { + const validatorMode = opts.validatorMode === 'shadow' ? 'shadow' : 'enforce'; + const sanitize = typeof opts.sanitizeTitle === 'function' ? opts.sanitizeTitle : (t) => t; + const sourceFromStory = typeof opts.sourceFromStory === 'function' ? opts.sourceFromStory : () => null; + + if (!Array.isArray(topStories) || topStories.length === 0) return null; + // Editorial gate: same bar the legacy pickBriefCluster enforced. + if (!topStories.some(isBriefLeadEligible)) return null; + + const parsed = parseBriefSynthesis(rawText, topStories.length); + if (!parsed) return null; + + const groundingStories = topStories.map((story) => ({ headline: story.primaryTitle })); + const storyGroundText = (story) => + [story.primaryTitle, ...(Array.isArray(story.memberTitles) ? story.memberTitles : [])].join(' — '); + + // Lead gates (#4928 external review — citation-SCOPED, not corpus-wide): + // every lead sentence must carry at least one citation, and its proper + // nouns must ground against ONLY the stories it cites. Corpus-wide + // validation let a claim bind to [1] while its facts came from story 3 + // — shape-valid misattribution. Anchor grounding stays as the overall + // floor. Any lead-level failure rejects to the legacy fallback. + let strippedCitations = 0; + const leadCheck = verifyCitationIndexes(parsed.lead, topStories.length); + strippedCitations += leadCheck.stripped; + const leadSentences = leadCheck.text.split(/(?<=[.!?])\s+/).filter((sentence) => sentence.trim().length > 0); + if (leadSentences.length === 0) return null; + for (const sentence of leadSentences) { + const cited = [...sentence.matchAll(/\[(\d{1,3})\]/g)] + .map((match) => Number.parseInt(match[1], 10)) + .filter((n) => n >= 1 && n <= topStories.length); + // Contract: every claim is cited. An uncited sentence is unverifiable. + if (cited.length === 0) return null; + const scopedGround = cited.map((n) => storyGroundText(topStories[n - 1])).join(' — '); + const sentenceValidation = validateNoHallucinatedProperNouns(sentence, scopedGround); + if (!sentenceValidation.ok && validatorMode === 'enforce') return null; + } + if (!checkLeadGrounding({ lead: leadCheck.text }, groundingStories, topStories.length)) return null; + + const lineByIndex = new Map(parsed.lines.map((line) => [line.n, line.text])); + let hallucinatedLines = 0; + const lines = topStories.map((story, i) => { + const n = i + 1; + const headline = sanitize(story.primaryTitle); + // Missing/degraded lines keep their citation so the contract + // ("every line ends with its own [n]") holds for renderers. + if (!lineByIndex.has(n)) return { n, text: `${headline} [${n}]` }; + // #4928 external review: a line for story n could carry [1] (or no + // citation at all after stripping) and the renderer would link the + // wrong source. The line's content is validated against story n, so + // its ONLY correct citation is [n]: strip every bracket marker and + // append the canonical one. + const bare = lineByIndex.get(n).replace(/\s*\[\d{1,3}\]/g, '').trim(); + const validation = validateNoHallucinatedProperNouns(bare, storyGroundText(story)); + if (!validation.ok) { + hallucinatedLines++; + if (validatorMode === 'enforce') return { n, text: `${headline} [${n}]` }; + } + return { n, text: `${bare} [${n}]` }; + }); + + // STRICT index lockstep: never filter — substitute. + const sources = topStories.map((story) => { + const source = sourceFromStory(story); + if (source) return source; + return { + title: sanitize(story.primaryTitle) || 'Untitled', + source: story.primarySource || 'Unknown', + url: '', + }; + }); + + return { lead: leadCheck.text, lines, sources, hallucinatedLines, strippedCitations }; +} diff --git a/scripts/lib/brief-llm.mjs b/scripts/lib/brief-llm.mjs index 9435260747..94198285b9 100644 --- a/scripts/lib/brief-llm.mjs +++ b/scripts/lib/brief-llm.mjs @@ -42,7 +42,13 @@ import { buildWhyMattersUserPrompt, hashBriefStory, parseWhyMatters, + checkLeadGrounding, + leadGroundsAgainstStory, } from '../../shared/brief-llm-core.js'; + +// #4921: the grounding spine now lives in shared/brief-llm-core.js — re-export +// for existing consumers of this module. +export { checkLeadGrounding, leadGroundsAgainstStory }; import { sanitizeForPrompt } from '../../server/_shared/llm-sanitize.js'; // Single source of truth for the brief story cap. Both buildDigestPrompt // and hashDigestInput must slice to this value or the LLM prose drifts @@ -505,273 +511,6 @@ export function buildDigestPrompt(stories, sensitivity, ctx = {}) { // Back-compat alias for tests that import the old constant name. export const DIGEST_PROSE_SYSTEM = DIGEST_PROSE_SYSTEM_BASE; -// Shared delimiter regex for tokenising both story headlines (anchor -// extraction) and synthesis prose (haystack lookup). Same delimiter -// set on both sides keeps the matching contract symmetric. -// -// Unicode quotes (U+2018, U+2019, U+201C, U+201D, U+00B4) are -// included alongside their ASCII counterparts. News headlines from -// Reuters/AP/Guardian use U+2019 for possessives ("China's", -// "Iran's", "DPRK's") and U+201C/U+201D for quoted phrases. Without -// splitting on them, "China's" becomes one token "china’s" that -// a lead saying "China" can never match — a false negative that -// would reject genuinely grounded leads. (PR #3667 review round 2 -// finding #2.) -const GROUNDING_TOKEN_DELIMS = /[\s,.!?;:()'"‘’“”´\\/—–\-[\]{}]+/; - -// Anchor-side stopword list. Story headlines often capitalise -// titles ("President Trump"), generic actors ("Officials confirmed"), -// quasi-adjectives ("Senior commander", "Federal court"), and -// sentence-start filler ("Following the announcement"). Without -// filtering, these enter storyTokens and a hallucinated lead like -// "President Biden announced..." passes the lead-anchor check via -// the shared word "President", then a teaser mentioning a real -// anchor satisfies the combined threshold — the visible top-of- -// email lead stays fabricated. (PR #3667 review round 2 finding #1.) -// -// Scope rule: only words that are commonly capitalised but do NOT -// discriminate a story. Specific entity names (people, places, -// orgs, brands) are NEVER on this list, even when common — "Iran", -// "Trump", "Israel", "EU", "UN" all stay in. "May" is also -// deliberately omitted (Theresa May, May Day, May = month all -// collide on it; safer to keep "may" matchable than to filter it -// and lose a real anchor). -// -// Maintenance heuristic (PR #3667 review round 5 #3): a capitalised -// token of length ≥4 belongs in this set if it appears in >~10% of -// real headlines without discriminating between stories. The cheap -// audit is: dump a week of headlines, tokenise with this same -// extractAnchorTokens function (with stopwords disabled), count -// frequencies, and inspect any token in >50 of ~500 headlines that -// isn't already a known proper noun. The "Prime"/"Chief"/"Cardinal" -// gaps caught on review rounds 2-3 would each have surfaced from -// such a frequency audit. Don't try to enumerate exhaustively up -// front; let production usage drive additions and capture each new -// ride-along bug class as a regression test. -const GROUNDING_ANCHOR_STOPWORDS = new Set([ - // Honorifics / titles - 'president', 'vice', 'senator', 'minister', 'secretary', - 'chairman', 'chairwoman', 'spokesman', 'spokeswoman', - 'director', 'general', 'admiral', 'colonel', 'captain', - 'mayor', 'governor', 'judge', 'justice', 'doctor', - 'professor', 'pope', 'rabbi', 'imam', 'sheikh', 'sultan', - 'emir', 'king', 'queen', 'prince', 'princess', - // Round-3 review additions: bigram-leading titles ("Prime - // Minister", "Chief Justice", "Cardinal Smith") whose first - // word alone passes the cap+length filter and would otherwise - // let a hallucinated "Prime Minister Trudeau announced..." lead - // ride on a "Prime Minister Netanyahu says..." headline via the - // shared "prime" token. PR #3667 review round 3. - 'prime', 'chief', 'premier', 'chancellor', 'speaker', - 'ambassador', 'envoy', 'commissioner', 'attorney', - 'cardinal', 'archbishop', 'monsignor', 'reverend', - 'pastor', 'bishop', 'lord', 'lady', 'dame', - 'congressman', 'congresswoman', 'congressperson', - 'representative', 'delegate', 'baron', 'baroness', - // Generic role plurals / institutional collectives - 'officials', 'officers', 'leaders', 'members', 'people', - 'forces', 'police', 'troops', 'agents', 'authorities', - 'sources', 'rebels', 'militants', 'protesters', 'civilians', - 'residents', 'citizens', 'workers', 'voters', - // Headline qualifiers / quasi-adjectives - 'senior', 'junior', 'former', 'acting', 'deputy', 'assistant', - 'federal', 'national', 'international', 'global', 'regional', - 'central', 'local', 'foreign', 'domestic', 'civil', 'public', - 'private', 'special', 'major', 'armed', - // Sentence-start / common filler - 'after', 'before', 'during', 'while', 'despite', 'following', - 'amid', 'today', 'yesterday', 'tomorrow', 'this', 'these', - 'those', 'when', 'where', 'what', 'which', 'breaking', - // News-headline glue - 'says', 'said', 'told', 'reports', 'analysis', 'opinion', - 'editorial', 'update', 'updates', - // Calendar (May omitted — see scope rule above) - 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', - 'saturday', 'sunday', 'january', 'february', 'march', 'april', - 'june', 'july', 'august', 'september', 'october', 'november', - 'december', -]); - -/** - * Anchor extraction from a story headline: capitalised + length ≥4 + - * NOT in GROUNDING_ANCHOR_STOPWORDS. The capitalisation filter makes - * this a "proper noun" heuristic; the stopword filter strips - * honorifics, role labels, bigram-leading titles, and sentence-start - * filler that would otherwise be shared anchors between any - * "President X..." headline and any "President Y..." hallucinated - * lead. File-level so the closure isn't re-instantiated per - * checkLeadGrounding call (PR #3667 review round 4 P2). - * - * @param {string} s - * @returns {string[]} lowercased anchor tokens - */ -function extractAnchorTokens(s) { - if (typeof s !== 'string' || s.length === 0) return []; - const out = []; - for (const w of s.split(GROUNDING_TOKEN_DELIMS)) { - if (w.length < 4 || !/^[A-Z]/.test(w)) continue; - const lower = w.toLowerCase(); - if (!GROUNDING_ANCHOR_STOPWORDS.has(lower)) out.push(lower); - } - return out; -} - -/** - * Tokenise synthesis prose into a Set of lowercased words for - * membership lookup. NO capitalisation filter — the synthesis can - * mention the entity in any case (sentence-medial, possessive form, - * etc.) and we still want it to count. File-level for the same - * reason as extractAnchorTokens (PR #3667 review round 4 P2). - * - * @param {string} text - * @returns {Set} - */ -function groundingTokenSet(text) { - const set = new Set(); - if (typeof text !== 'string' || text.length === 0) return set; - for (const w of text.toLowerCase().split(GROUNDING_TOKEN_DELIMS)) { - if (w.length >= 4) set.add(w); - } - return set; -} - -/** - * Cheap content-grounding check: the canonical lead MUST reference - * proper-noun tokens that actually appear in the input story - * headlines. Without this, the LLM is free to confabulate even with - * shape-valid output — e.g. the 2026-05-12 incident where a Trump- - * era geopolitics pool (Iran/Israel/Sudan/Cuba/Ukraine) shipped a - * "President Biden announced a crypto executive order" lead. Shape - * was valid; content was a complete fabrication the model produced - * from training-data priors instead of grounding. - * - * Two independent grounding requirements (BOTH must pass): - * - * 1. **Lead anchor**: the lead alone must hit ≥1 anchor token. - * Without this, a hallucinated lead can sneak through when the - * threads happen to mention real entities — the visible lead - * stays fabricated even though the combined check passes. - * (Code-review finding on PR #3667 #1.) - * 2. **Combined coverage**: the lead + thread teasers together - * must hit ≥2 anchors (relaxed to 1 when the corpus itself has - * <4 anchor tokens, so single-named-actor briefs aren't - * false-positives). - * - * Matching is **token-set membership** — both sides are split on - * the same delimiter regex and lowercased into Sets. Substring - * matching (the v1 implementation) was rejected on PR #3667 review: - * it accepts unrelated entities like `iran` inside `tirana`, - * `oman` inside `romania`, `india` inside `indiana`. Token-set - * matching avoids that class of false positive cleanly. - * (Code-review finding on PR #3667 #2.) - * - * Length cap of 4 deliberately filters out 2-letter ISO country - * codes (`IR`, `PS`, `US`) and short-form orgs (`UN`, `EU`, `RSF`) - * which are too generic to be discriminating anchors. The check is - * about whether the lead names a SPECIFIC entity — not whether it - * uses any capitalised token at all. - * - * Returns true (grounded, or check-skipped because corpus lacks - * signal / no stories supplied) → accept. Returns false → reject. - * - * @param {{ lead?: string; threads?: Array<{tag?:string;teaser?:string}> }} synthesis - * @param {Array<{ headline?: string }>} stories - * @returns {boolean} - */ -export function checkLeadGrounding(synthesis, stories) { - if (!Array.isArray(stories) || stories.length === 0) return true; - - const storyTokens = new Set(); - for (const s of stories.slice(0, MAX_STORIES_PER_USER)) { - for (const tok of extractAnchorTokens(s?.headline ?? '')) { - storyTokens.add(tok); - } - } - // Corpus has no proper-noun anchors — can't validate, skip. - // Genuine input (2026-era stories) reliably has >0 such tokens; - // the empty branch is for synthetic / single-headline tests. - // - // Lowercase-headline blind spot (PR #3667 review round 5 #2): - // if a feed ever produces all-lowercase or all-≤3-char headlines, - // every story contributes zero anchors and the gate silently - // skips. Emit a warn so ops can detect the regression — but only - // when stories.length is meaningful (≥3) so the synthetic - // single-headline test corpora don't spam logs. - if (storyTokens.size === 0) { - if (stories.length >= 3) { - console.warn( - `[brief-llm] grounding gate skipped: storyTokens empty for stories.length=${stories.length} — likely all-lowercase or <4-char headlines from a feed regression`, - ); - } - return true; - } - - const leadTokens = groundingTokenSet(typeof synthesis?.lead === 'string' ? synthesis.lead : ''); - - // Requirement 1: the lead alone must hit ≥1 anchor. A hallucinated - // lead with grounded teasers would otherwise pass — the user still - // sees the fabricated text at the top of the email. - let leadHasAnchor = false; - for (const tok of leadTokens) { - if (storyTokens.has(tok)) { leadHasAnchor = true; break; } - } - if (!leadHasAnchor) return false; - - // Requirement 2: combined lead + teasers hit ≥threshold anchors. - // Threshold relaxes to 1 when the corpus is sparse so single- - // story briefs don't false-positive. - const combinedTokens = new Set(leadTokens); - for (const t of (Array.isArray(synthesis?.threads) ? synthesis.threads : [])) { - for (const w of groundingTokenSet(typeof t?.teaser === 'string' ? t.teaser : '')) { - combinedTokens.add(w); - } - } - const threshold = storyTokens.size >= 4 ? 2 : 1; - let combinedHits = 0; - for (const tok of storyTokens) { - if (combinedTokens.has(tok)) { - combinedHits++; - if (combinedHits >= threshold) return true; - } - } - return false; -} - -/** - * Lead ↔ single-story coherence check (F4). Returns true iff `lead` - * shares ≥1 proper-noun anchor with `headline`. Reuses the same - * anchor machinery as `checkLeadGrounding` (capitalised, length ≥4, - * stopword-filtered headline anchors; token-set membership against - * the lead) but with a FIXED threshold of 1 — coherence asks only - * "is the lead about the same story?", not "how well-grounded is it?". - * - * `checkLeadGrounding` itself is the wrong fit here: scoped to one - * story, a single headline can carry ≥4 anchor tokens, which trips - * its `size >= 4 ? 2 : 1` threshold up to 2 — too strict for - * coherence, where a lead legitimately about card #1 may name only - * one of its entities. - * - * Used by the cron's lead/card-#1 coherence telemetry - * (`composeAndStoreBriefForUser`) — see plan - * docs/plans/2026-05-14-001-…-plan.md (F4, Phase 4). - * - * @param {string} lead — the canonical synthesis lead - * @param {string} headline — the rendered first card's headline - * @returns {boolean} true = coherent (or check-skipped); false = the - * lead names none of the headline's proper-noun anchors - */ -export function leadGroundsAgainstStory(lead, headline) { - const anchors = new Set(extractAnchorTokens(typeof headline === 'string' ? headline : '')); - // No proper-noun anchors in the headline → cannot judge coherence, - // skip (same "degenerate corpus → accept" stance as checkLeadGrounding). - if (anchors.size === 0) return true; - const leadTokens = groundingTokenSet(typeof lead === 'string' ? lead : ''); - for (const tok of anchors) { - if (leadTokens.has(tok)) return true; - } - return false; -} - /** * Strict shape check for a parsed digest-prose object. Used by BOTH * parseDigestProse (fresh LLM output) AND generateDigestProse's @@ -851,7 +590,7 @@ export function validateDigestProseShape(obj, stories) { // see — checkLeadGrounding inspects `lead` and `threads[].teaser`, // both already trimmed and capped above. if (Array.isArray(stories) && stories.length > 0 - && !checkLeadGrounding({ lead, threads }, stories)) { + && !checkLeadGrounding({ lead, threads }, stories, MAX_STORIES_PER_USER)) { return null; } diff --git a/scripts/seed-insights.mjs b/scripts/seed-insights.mjs index de4b7ea2c2..ce0f4d64b6 100644 --- a/scripts/seed-insights.mjs +++ b/scripts/seed-insights.mjs @@ -10,7 +10,14 @@ import { } from './_clustering.mjs'; import { extractCountryCode } from './shared/geo-extract.mjs'; import { unwrapEnvelope } from './_seed-envelope-source.mjs'; -import { pickBriefCluster, briefSystemPrompt, briefUserPrompt } from './_insights-brief.mjs'; +import { + pickBriefCluster, + briefSystemPrompt, + briefUserPrompt, + synthesisSystemPrompt, + synthesisUserPrompt, + composeSynthesizedBrief, +} from './_insights-brief.mjs'; // Import from the scripts mirror (`scripts/shared/`) — NOT the repo-root // `shared/`. Railway services with nixpacks `rootDirectory=scripts` only // package files under scripts/; a `../shared/` import resolves to @@ -25,8 +32,11 @@ import { validateNoHallucinatedProperNouns } from './shared/brief-llm-core.js'; // output unchanged (default, safe). `enforce` = on violation, replace // the LLM summary with the source headline. Flip via Railway env after // the 7-day shadow window confirms <5% violation rate. +// #4921: enforce is the DEFAULT — the shadow window measured its +// false-positive rate; shipping detected hallucinations was the residual +// risk. Set BRIEF_VALIDATOR_MODE=shadow to revert during an incident. const BRIEF_VALIDATOR_MODE = - process.env.BRIEF_VALIDATOR_MODE === 'enforce' ? 'enforce' : 'shadow'; + process.env.BRIEF_VALIDATOR_MODE === 'shadow' ? 'shadow' : 'enforce'; // True only when run directly as a cron entry (node seed-insights.mjs), false // when imported by tests — so importing the module doesn't load .env or fire a @@ -119,6 +129,54 @@ function briefSourceFromStory(story) { return publishedAt ? { title, source, url, publishedAt } : { title, source, url }; } +/** + * #4928: the legacy single-headline brief, extracted intact from the main + * flow (L2 of the fallback chain). Corroboration-gated via + * pickBriefCluster; enforce/shadow semantics unchanged. + */ +async function generateLegacySingleHeadlineBrief(topStories) { + const briefCluster = pickBriefCluster(topStories); + const topHeadline = briefCluster ? sanitizeTitle(briefCluster.primaryTitle) : ''; + const worldBriefSources = briefCluster ? [briefSourceFromStory(briefCluster)].filter(Boolean) : []; + + if (!topHeadline) { + console.warn(' No multi-source cluster available — publishing degraded (stories without brief)'); + return { worldBrief: '', briefProvider: '', briefModel: '', worldBriefSources, status: 'degraded' }; + } + + const llmResult = await callLLM(topHeadline); + if (!llmResult) { + console.warn(' No LLM available — publishing degraded (stories without brief)'); + return { worldBrief: '', briefProvider: '', briefModel: '', worldBriefSources, status: 'degraded' }; + } + + // Hallucination check: did the LLM invent proper nouns not in the + // headline? (May 19 incident: "Lebanese President Michel Aoun pledged…" + // against a nameless headline. docs/plans/2026-05-19-001 U2.) + const validation = validateNoHallucinatedProperNouns(llmResult.text, topHeadline); + if (!validation.ok) { + const hallucinated = (validation.hallucinated || []).join(' '); + if (BRIEF_VALIDATOR_MODE === 'enforce') { + console.warn(` [brief_hallucination ENFORCE] dropped LLM summary: invented "${hallucinated}" not in headline; fell back to headline`); + return { + worldBrief: topHeadline, + briefProvider: `${llmResult.provider}+headline-fallback`, + briefModel: llmResult.model, + worldBriefSources, + status: 'ok', + }; + } + console.warn(` [brief_hallucination SHADOW] would have dropped LLM summary: invented "${hallucinated}" not in headline`); + } + return { + worldBrief: llmResult.text, + briefProvider: llmResult.provider, + briefModel: llmResult.model, + worldBriefSources, + status: 'ok', + }; +} + async function readDigestFromRedis() { const { url, token } = getRedisCredentials(); const resp = await fetch(`${url}/get/${encodeURIComponent(DIGEST_KEY)}`, { @@ -192,8 +250,13 @@ function __setInsightsLlmTransportForTests(overrides = null) { } async function callLLM(headline, options = {}) { - const systemPrompt = briefSystemPrompt(new Date().toISOString().split('T')[0]); - const userPrompt = briefUserPrompt(headline); + // #4921: callers may supply explicit prompts (the top-8 synthesis call); + // the headline default keeps the legacy single-headline path and its + // retry tests unchanged. + const systemPrompt = options.systemPrompt + ?? briefSystemPrompt(new Date().toISOString().split('T')[0]); + const userPrompt = options.userPrompt ?? briefUserPrompt(headline); + const maxTokens = Number.isFinite(options.maxTokens) ? options.maxTokens : 300; const insightsFetch = insightsLlmFetchForTests || ((...args) => globalThis.fetch(...args)); const callBudgetMs = Number.isFinite(options.callBudgetMs) @@ -225,7 +288,7 @@ async function callLLM(headline, options = {}) { { role: 'system', content: systemPrompt }, { role: 'user', content: userPrompt }, ], - max_tokens: 300, + max_tokens: maxTokens, temperature: 0.1, ...provider.extraBody, }), @@ -419,59 +482,56 @@ async function fetchInsights() { // continues to include single-source clusters, rendered as the headline list // under the brief. The brief paragraph is the one surface where corroboration // matters; the list is already visually marked with per-story sourceCount. - const briefCluster = pickBriefCluster(topStories); - const topHeadline = briefCluster ? sanitizeTitle(briefCluster.primaryTitle) : ''; - const worldBriefSources = briefCluster ? [briefSourceFromStory(briefCluster)].filter(Boolean) : []; - + // #4921/#4928: L1 = top-8 synthesis via the pure composer (parse + + // corroboration gate + lead noun/anchor gates + per-line enforcement + + // citation verification + index-locked sources — all unit-tested in + // _insights-brief.mjs). L2 = legacy single-headline brief. Degraded last. + // The brief always ships. let worldBrief = ''; let briefProvider = ''; let briefModel = ''; + let briefStoryLines = []; + let worldBriefSources = []; let status = 'ok'; - if (!topHeadline) { - status = 'degraded'; - console.warn(' No multi-source cluster available — publishing degraded (stories without brief)'); + const synthesisResult = topStories.length > 0 + ? await callLLM(null, { + systemPrompt: synthesisSystemPrompt(new Date().toISOString().split('T')[0]), + userPrompt: synthesisUserPrompt(topStories), + maxTokens: 900, + }) + : null; + const composed = synthesisResult + ? composeSynthesizedBrief(synthesisResult.text, topStories, { + validatorMode: BRIEF_VALIDATOR_MODE, + sanitizeTitle, + sourceFromStory: briefSourceFromStory, + }) + : null; + + if (composed) { + worldBrief = composed.lead; + briefStoryLines = composed.lines; + worldBriefSources = composed.sources; + briefProvider = synthesisResult.provider; + briefModel = synthesisResult.model; + if (composed.strippedCitations > 0) { + console.warn(` [brief_citation ENFORCE] stripped ${composed.strippedCitations} out-of-range citation(s)`); + } + if (composed.hallucinatedLines > 0) { + console.warn(` [brief_hallucination ${BRIEF_VALIDATOR_MODE.toUpperCase()}] ${composed.hallucinatedLines}/${topStories.length} synthesis lines flagged`); + } + console.log(` Brief synthesized (top-${topStories.length}) via ${briefProvider} (${briefModel})`); } else { - const llmResult = await callLLM(topHeadline); - if (llmResult) { - // Hallucination check: did the LLM invent proper nouns not in - // the headline? The May 19 brief shipped "Lebanese President - // Michel Aoun pledged..." against a headline that contained no - // name. See docs/plans/2026-05-19-001 U2. - const validation = validateNoHallucinatedProperNouns(llmResult.text, topHeadline); - if (!validation.ok) { - const hallucinated = (validation.hallucinated || []).join(' '); - if (BRIEF_VALIDATOR_MODE === 'enforce') { - // Replace the LLM summary with the source headline. R1 of the - // plan: "falls back to a safe summary (headline-grounded - // template) rather than publishing the hallucination." - worldBrief = topHeadline; - briefProvider = `${llmResult.provider}+headline-fallback`; - briefModel = llmResult.model; - console.warn( - ` [brief_hallucination ENFORCE] dropped LLM summary: invented "${hallucinated}" not in headline; fell back to headline` - ); - } else { - // Shadow mode: log but ship the LLM output. The 7-day rollout - // window measures the false-positive rate before flipping to - // enforce. - worldBrief = llmResult.text; - briefProvider = llmResult.provider; - briefModel = llmResult.model; - console.warn( - ` [brief_hallucination SHADOW] would have dropped LLM summary: invented "${hallucinated}" not in headline` - ); - } - } else { - worldBrief = llmResult.text; - briefProvider = llmResult.provider; - briefModel = llmResult.model; - console.log(` Brief generated via ${briefProvider} (${briefModel})`); - } - } else { - status = 'degraded'; - console.warn(' No LLM available — publishing degraded (stories without brief)'); + if (synthesisResult) { + console.warn(' [brief_synthesis] composer rejected output (parse/gates) — falling back to single-headline brief'); } + const legacy = await generateLegacySingleHeadlineBrief(topStories); + worldBrief = legacy.worldBrief; + briefProvider = legacy.briefProvider; + briefModel = legacy.briefModel; + worldBriefSources = legacy.worldBriefSources; + status = legacy.status; } const multiSourceCount = clusters.filter(c => (c.sources?.length ?? 0) >= 2 || c.entityCorroboration === true).length; @@ -526,8 +586,21 @@ async function fetchInsights() { `drops adm=${provenance.selectionDrops.admissibility} srcCap=${provenance.selectionDrops.sourceCap} overflow=${provenance.selectionDrops.overflow}`, ); + // #4921 staleness footer: the age window of the BRIEF'S OWN material — + // the top stories the synthesis cites — not the whole digest pool + // (#4928 external review: an unrelated fresh item made the footer claim + // the brief's sources were fresher than they are). + const pubTimes = topStories + .map(story => new Date(story.pubDate).getTime()) + .filter(Number.isFinite); + const sourceAgeRange = pubTimes.length > 0 + ? { newestMs: Math.max(...pubTimes), oldestMs: Math.min(...pubTimes) } + : null; + const payload = { worldBrief, + briefStoryLines, + sourceAgeRange, worldBriefSources, briefProvider, briefModel, diff --git a/scripts/shared/brief-llm-core.d.ts b/scripts/shared/brief-llm-core.d.ts index d58f487e79..af48406459 100644 --- a/scripts/shared/brief-llm-core.d.ts +++ b/scripts/shared/brief-llm-core.d.ts @@ -43,3 +43,17 @@ export function validateNoHallucinatedProperNouns( summary: unknown, headline: unknown, ): { ok: true } | { ok: false; hallucinated: string[] }; + +// ── Grounding spine (#4921) ──────────────────────────────────────────────── +export function extractAnchorTokens(s: string): string[]; +export function groundingTokenSet(text: string): Set; +export function checkLeadGrounding( + synthesis: { lead?: string; threads?: Array<{ tag?: string; teaser?: string }> }, + stories: Array<{ headline?: string }>, + storyCap?: number, +): boolean; +export function leadGroundsAgainstStory(lead: string, headline: string): boolean; +export function verifyCitationIndexes( + text: string, + sourceCount: number, +): { text: string; stripped: number }; diff --git a/scripts/shared/brief-llm-core.js b/scripts/shared/brief-llm-core.js index b1c6fa05aa..0d33b2b44d 100644 --- a/scripts/shared/brief-llm-core.js +++ b/scripts/shared/brief-llm-core.js @@ -101,7 +101,9 @@ export function parseWhyMatters(text) { /** * Deterministic 16-char hex hash of the SIX story fields that flow - * into the whyMatters prompt (5 core + description). Cache identity + * into the whyMatters prompt (5 core + description). Also consumed by + * server/worldmonitor/intelligence/v1/get-country-intel-brief.ts + * (citation verification + grounding telemetry, #4921). Cache identity * MUST cover every field that shapes the LLM output, or two requests * with the same core fields but different descriptions will share a * cache entry and the second caller gets prose grounded in the first @@ -658,3 +660,316 @@ function containsSubsequence(haystack, needle) { } return false; } + +// ═══════════════════════════════════════════════════════════════════════════ +// Grounding spine (#4921 — ported from scripts/lib/brief-llm.mjs so EVERY +// brief product shares one grounding implementation; brief-llm.mjs +// re-exports these for backcompat). Edge-safe: no node:crypto, no env reads. +// ═══════════════════════════════════════════════════════════════════════════ + + +// Shared delimiter regex for tokenising both story headlines (anchor +// extraction) and synthesis prose (haystack lookup). Same delimiter +// set on both sides keeps the matching contract symmetric. +// +// Unicode quotes (U+2018, U+2019, U+201C, U+201D, U+00B4) are +// included alongside their ASCII counterparts. News headlines from +// Reuters/AP/Guardian use U+2019 for possessives ("China's", +// "Iran's", "DPRK's") and U+201C/U+201D for quoted phrases. Without +// splitting on them, "China's" becomes one token "china’s" that +// a lead saying "China" can never match — a false negative that +// would reject genuinely grounded leads. (PR #3667 review round 2 +// finding #2.) +const GROUNDING_TOKEN_DELIMS = /[\s,.!?;:()'"‘’“”´\\/—–\-[\]{}]+/; + +// Anchor-side stopword list. Story headlines often capitalise +// titles ("President Trump"), generic actors ("Officials confirmed"), +// quasi-adjectives ("Senior commander", "Federal court"), and +// sentence-start filler ("Following the announcement"). Without +// filtering, these enter storyTokens and a hallucinated lead like +// "President Biden announced..." passes the lead-anchor check via +// the shared word "President", then a teaser mentioning a real +// anchor satisfies the combined threshold — the visible top-of- +// email lead stays fabricated. (PR #3667 review round 2 finding #1.) +// +// Scope rule: only words that are commonly capitalised but do NOT +// discriminate a story. Specific entity names (people, places, +// orgs, brands) are NEVER on this list, even when common — "Iran", +// "Trump", "Israel", "EU", "UN" all stay in. "May" is also +// deliberately omitted (Theresa May, May Day, May = month all +// collide on it; safer to keep "may" matchable than to filter it +// and lose a real anchor). +// +// Maintenance heuristic (PR #3667 review round 5 #3): a capitalised +// token of length ≥4 belongs in this set if it appears in >~10% of +// real headlines without discriminating between stories. The cheap +// audit is: dump a week of headlines, tokenise with this same +// extractAnchorTokens function (with stopwords disabled), count +// frequencies, and inspect any token in >50 of ~500 headlines that +// isn't already a known proper noun. The "Prime"/"Chief"/"Cardinal" +// gaps caught on review rounds 2-3 would each have surfaced from +// such a frequency audit. Don't try to enumerate exhaustively up +// front; let production usage drive additions and capture each new +// ride-along bug class as a regression test. +const GROUNDING_ANCHOR_STOPWORDS = new Set([ + // Honorifics / titles + 'president', 'vice', 'senator', 'minister', 'secretary', + 'chairman', 'chairwoman', 'spokesman', 'spokeswoman', + 'director', 'general', 'admiral', 'colonel', 'captain', + 'mayor', 'governor', 'judge', 'justice', 'doctor', + 'professor', 'pope', 'rabbi', 'imam', 'sheikh', 'sultan', + 'emir', 'king', 'queen', 'prince', 'princess', + // Round-3 review additions: bigram-leading titles ("Prime + // Minister", "Chief Justice", "Cardinal Smith") whose first + // word alone passes the cap+length filter and would otherwise + // let a hallucinated "Prime Minister Trudeau announced..." lead + // ride on a "Prime Minister Netanyahu says..." headline via the + // shared "prime" token. PR #3667 review round 3. + 'prime', 'chief', 'premier', 'chancellor', 'speaker', + 'ambassador', 'envoy', 'commissioner', 'attorney', + 'cardinal', 'archbishop', 'monsignor', 'reverend', + 'pastor', 'bishop', 'lord', 'lady', 'dame', + 'congressman', 'congresswoman', 'congressperson', + 'representative', 'delegate', 'baron', 'baroness', + // Generic role plurals / institutional collectives + 'officials', 'officers', 'leaders', 'members', 'people', + 'forces', 'police', 'troops', 'agents', 'authorities', + 'sources', 'rebels', 'militants', 'protesters', 'civilians', + 'residents', 'citizens', 'workers', 'voters', + // Headline qualifiers / quasi-adjectives + 'senior', 'junior', 'former', 'acting', 'deputy', 'assistant', + 'federal', 'national', 'international', 'global', 'regional', + 'central', 'local', 'foreign', 'domestic', 'civil', 'public', + 'private', 'special', 'major', 'armed', + // Sentence-start / common filler + 'after', 'before', 'during', 'while', 'despite', 'following', + 'amid', 'today', 'yesterday', 'tomorrow', 'this', 'these', + 'those', 'when', 'where', 'what', 'which', 'breaking', + // News-headline glue + 'says', 'said', 'told', 'reports', 'analysis', 'opinion', + 'editorial', 'update', 'updates', + // Calendar (May omitted — see scope rule above) + 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', + 'saturday', 'sunday', 'january', 'february', 'march', 'april', + 'june', 'july', 'august', 'september', 'october', 'november', + 'december', +]); + +/** + * Anchor extraction from a story headline: capitalised + length ≥4 + + * NOT in GROUNDING_ANCHOR_STOPWORDS. The capitalisation filter makes + * this a "proper noun" heuristic; the stopword filter strips + * honorifics, role labels, bigram-leading titles, and sentence-start + * filler that would otherwise be shared anchors between any + * "President X..." headline and any "President Y..." hallucinated + * lead. File-level so the closure isn't re-instantiated per + * checkLeadGrounding call (PR #3667 review round 4 P2). + * + * @param {string} s + * @returns {string[]} lowercased anchor tokens + */ +export function extractAnchorTokens(s) { + if (typeof s !== 'string' || s.length === 0) return []; + const out = []; + for (const w of s.split(GROUNDING_TOKEN_DELIMS)) { + if (w.length < 4 || !/^[A-Z]/.test(w)) continue; + const lower = w.toLowerCase(); + if (!GROUNDING_ANCHOR_STOPWORDS.has(lower)) out.push(lower); + } + return out; +} + +/** + * Tokenise synthesis prose into a Set of lowercased words for + * membership lookup. NO capitalisation filter — the synthesis can + * mention the entity in any case (sentence-medial, possessive form, + * etc.) and we still want it to count. File-level for the same + * reason as extractAnchorTokens (PR #3667 review round 4 P2). + * + * @param {string} text + * @returns {Set} + */ +export function groundingTokenSet(text) { + const set = new Set(); + if (typeof text !== 'string' || text.length === 0) return set; + for (const w of text.toLowerCase().split(GROUNDING_TOKEN_DELIMS)) { + if (w.length >= 4) set.add(w); + } + return set; +} + +/** + * Cheap content-grounding check: the canonical lead MUST reference + * proper-noun tokens that actually appear in the input story + * headlines. Without this, the LLM is free to confabulate even with + * shape-valid output — e.g. the 2026-05-12 incident where a Trump- + * era geopolitics pool (Iran/Israel/Sudan/Cuba/Ukraine) shipped a + * "President Biden announced a crypto executive order" lead. Shape + * was valid; content was a complete fabrication the model produced + * from training-data priors instead of grounding. + * + * Two independent grounding requirements (BOTH must pass): + * + * 1. **Lead anchor**: the lead alone must hit ≥1 anchor token. + * Without this, a hallucinated lead can sneak through when the + * threads happen to mention real entities — the visible lead + * stays fabricated even though the combined check passes. + * (Code-review finding on PR #3667 #1.) + * 2. **Combined coverage**: the lead + thread teasers together + * must hit ≥2 anchors (relaxed to 1 when the corpus itself has + * <4 anchor tokens, so single-named-actor briefs aren't + * false-positives). + * + * Matching is **token-set membership** — both sides are split on + * the same delimiter regex and lowercased into Sets. Substring + * matching (the v1 implementation) was rejected on PR #3667 review: + * it accepts unrelated entities like `iran` inside `tirana`, + * `oman` inside `romania`, `india` inside `indiana`. Token-set + * matching avoids that class of false positive cleanly. + * (Code-review finding on PR #3667 #2.) + * + * Length cap of 4 deliberately filters out 2-letter ISO country + * codes (`IR`, `PS`, `US`) and short-form orgs (`UN`, `EU`, `RSF`) + * which are too generic to be discriminating anchors. The check is + * about whether the lead names a SPECIFIC entity — not whether it + * uses any capitalised token at all. + * + * Returns true (grounded, or check-skipped because corpus lacks + * signal / no stories supplied) → accept. Returns false → reject. + * + * @param {{ lead?: string; threads?: Array<{tag?:string;teaser?:string}> }} synthesis + * @param {Array<{ headline?: string }>} stories + * @returns {boolean} + */ +// Default story cap for grounding: mirrors the digest's default +// MAX_STORIES_PER_USER without dragging its env read into this +// edge-safe module — callers with a different cap pass it explicitly. +const DEFAULT_GROUNDING_STORY_CAP = 8; + +export function checkLeadGrounding(synthesis, stories, storyCap = DEFAULT_GROUNDING_STORY_CAP) { + if (!Array.isArray(stories) || stories.length === 0) return true; + + const storyTokens = new Set(); + for (const s of stories.slice(0, storyCap)) { + for (const tok of extractAnchorTokens(s?.headline ?? '')) { + storyTokens.add(tok); + } + } + // Corpus has no proper-noun anchors — can't validate, skip. + // Genuine input (2026-era stories) reliably has >0 such tokens; + // the empty branch is for synthetic / single-headline tests. + // + // Lowercase-headline blind spot (PR #3667 review round 5 #2): + // if a feed ever produces all-lowercase or all-≤3-char headlines, + // every story contributes zero anchors and the gate silently + // skips. Emit a warn so ops can detect the regression — but only + // when stories.length is meaningful (≥3) so the synthetic + // single-headline test corpora don't spam logs. + if (storyTokens.size === 0) { + if (stories.length >= 3) { + console.warn( + `[brief-llm-core] grounding gate skipped: storyTokens empty for stories.length=${stories.length} — likely all-lowercase or <4-char headlines from a feed regression`, + ); + } + return true; + } + + const leadTokens = groundingTokenSet(typeof synthesis?.lead === 'string' ? synthesis.lead : ''); + + // Requirement 1: the lead alone must hit ≥1 anchor. A hallucinated + // lead with grounded teasers would otherwise pass — the user still + // sees the fabricated text at the top of the email. + let leadHasAnchor = false; + for (const tok of leadTokens) { + if (storyTokens.has(tok)) { leadHasAnchor = true; break; } + } + if (!leadHasAnchor) return false; + + // Requirement 2: combined lead + teasers hit ≥threshold anchors. + // Threshold relaxes to 1 when the corpus is sparse so single- + // story briefs don't false-positive. + const combinedTokens = new Set(leadTokens); + for (const t of (Array.isArray(synthesis?.threads) ? synthesis.threads : [])) { + for (const w of groundingTokenSet(typeof t?.teaser === 'string' ? t.teaser : '')) { + combinedTokens.add(w); + } + } + const threshold = storyTokens.size >= 4 ? 2 : 1; + let combinedHits = 0; + for (const tok of storyTokens) { + if (combinedTokens.has(tok)) { + combinedHits++; + if (combinedHits >= threshold) return true; + } + } + return false; +} + +/** + * Lead ↔ single-story coherence check (F4). Returns true iff `lead` + * shares ≥1 proper-noun anchor with `headline`. Reuses the same + * anchor machinery as `checkLeadGrounding` (capitalised, length ≥4, + * stopword-filtered headline anchors; token-set membership against + * the lead) but with a FIXED threshold of 1 — coherence asks only + * "is the lead about the same story?", not "how well-grounded is it?". + * + * `checkLeadGrounding` itself is the wrong fit here: scoped to one + * story, a single headline can carry ≥4 anchor tokens, which trips + * its `size >= 4 ? 2 : 1` threshold up to 2 — too strict for + * coherence, where a lead legitimately about card #1 may name only + * one of its entities. + * + * Used by the cron's lead/card-#1 coherence telemetry + * (`composeAndStoreBriefForUser`) — see plan + * docs/plans/2026-05-14-001-…-plan.md (F4, Phase 4). + * + * @param {string} lead — the canonical synthesis lead + * @param {string} headline — the rendered first card's headline + * @returns {boolean} true = coherent (or check-skipped); false = the + * lead names none of the headline's proper-noun anchors + */ +export function leadGroundsAgainstStory(lead, headline) { + const anchors = new Set(extractAnchorTokens(typeof headline === 'string' ? headline : '')); + // No proper-noun anchors in the headline → cannot judge coherence, + // skip (same "degenerate corpus → accept" stance as checkLeadGrounding). + if (anchors.size === 0) return true; + const leadTokens = groundingTokenSet(typeof lead === 'string' ? lead : ''); + for (const tok of anchors) { + if (leadTokens.has(tok)) return true; + } + return false; +} + + +/** + * #4921: mechanical citation verification. Every bracket citation [n] + * in LLM brief prose must map to a real entry in the grounding source + * list (1..sourceCount). Out-of-range markers are invented — strip them + * rather than rendering a dead reference. Returns the cleaned text and + * the count of stripped markers (callers log/telemeter when > 0). + * + * Pure string operation; deliberately does NOT try to verify the CLAIM + * against the source — that is the grounding gates' job. This closes + * the cheaper hole: "[9]" shipped against a 6-source list. + * + * @param {string} text + * @param {number} sourceCount + * @returns {{ text: string; stripped: number }} + */ +export function verifyCitationIndexes(text, sourceCount) { + if (typeof text !== 'string' || text.length === 0) { + return { text: typeof text === 'string' ? text : '', stripped: 0 }; + } + const max = Number.isFinite(sourceCount) && sourceCount > 0 ? Math.floor(sourceCount) : 0; + let stripped = 0; + // 1-3 digits: [123] must not sail through unverified (review finding); + // 4+ digit brackets ([2026]) are treated as prose, not citations. + const cleaned = text.replace(/\s*\[(\d{1,3})\]/g, (full, numStr) => { + const n = Number.parseInt(numStr, 10); + if (n >= 1 && n <= max) return full; + stripped++; + return ''; + }); + return { text: cleaned, stripped }; +} + diff --git a/server/worldmonitor/intelligence/v1/get-country-intel-brief.ts b/server/worldmonitor/intelligence/v1/get-country-intel-brief.ts index c77017c2e9..dec80450a5 100644 --- a/server/worldmonitor/intelligence/v1/get-country-intel-brief.ts +++ b/server/worldmonitor/intelligence/v1/get-country-intel-brief.ts @@ -8,6 +8,7 @@ import type { import { cachedFetchJson, getCachedJson } from '../../../_shared/redis'; import { UPSTREAM_TIMEOUT_MS, TIER1_COUNTRIES, sha256Hex } from './_shared'; import { callLlm } from '../../../_shared/llm'; +import { verifyCitationIndexes, checkLeadGrounding } from '../../../../shared/brief-llm-core.js'; import { isCallerPremium } from '../../../_shared/premium-check'; import { sanitizeForPrompt } from '../../../_shared/llm-sanitize.js'; import { ENERGY_SPINE_KEY_PREFIX } from '../../../_shared/cache-keys'; @@ -239,10 +240,33 @@ Rules: if (!llmResult) return null; + // #4921 brief contract: citations are verified mechanically — every + // [n] must map to a real grounding source; invented indexes are + // stripped before shipping (ENFORCE). The prompt demands "do not + // invent source numbers", but demands are not guarantees. + const citationCheck = verifyCitationIndexes(llmResult.content, entrySources.length); + if (citationCheck.stripped > 0) { + console.warn( + `[country-intel] stripped ${citationCheck.stripped} out-of-range citation(s) ` + + `for ${req.countryCode} (sources=${entrySources.length})`, + ); + } + // Grounding telemetry (measure-only for now — the analyst format + // legitimately synthesizes across sources, so enforce here needs its + // own false-positive window first; see #4921). + const grounded = checkLeadGrounding( + { lead: citationCheck.text.slice(0, 600) }, + entrySources.map((source) => ({ headline: source.title })), + entrySources.length || 1, + ); + if (!grounded) { + console.warn(`[country-intel] GROUNDING MEASURE: brief for ${req.countryCode} names no source anchor`); + } + return { countryCode: req.countryCode, countryName, - brief: llmResult.content, + brief: citationCheck.text, model: llmResult.model, generatedAt: Date.now(), sources: entrySources, diff --git a/shared/brief-llm-core.d.ts b/shared/brief-llm-core.d.ts index d58f487e79..af48406459 100644 --- a/shared/brief-llm-core.d.ts +++ b/shared/brief-llm-core.d.ts @@ -43,3 +43,17 @@ export function validateNoHallucinatedProperNouns( summary: unknown, headline: unknown, ): { ok: true } | { ok: false; hallucinated: string[] }; + +// ── Grounding spine (#4921) ──────────────────────────────────────────────── +export function extractAnchorTokens(s: string): string[]; +export function groundingTokenSet(text: string): Set; +export function checkLeadGrounding( + synthesis: { lead?: string; threads?: Array<{ tag?: string; teaser?: string }> }, + stories: Array<{ headline?: string }>, + storyCap?: number, +): boolean; +export function leadGroundsAgainstStory(lead: string, headline: string): boolean; +export function verifyCitationIndexes( + text: string, + sourceCount: number, +): { text: string; stripped: number }; diff --git a/shared/brief-llm-core.js b/shared/brief-llm-core.js index b1c6fa05aa..0d33b2b44d 100644 --- a/shared/brief-llm-core.js +++ b/shared/brief-llm-core.js @@ -101,7 +101,9 @@ export function parseWhyMatters(text) { /** * Deterministic 16-char hex hash of the SIX story fields that flow - * into the whyMatters prompt (5 core + description). Cache identity + * into the whyMatters prompt (5 core + description). Also consumed by + * server/worldmonitor/intelligence/v1/get-country-intel-brief.ts + * (citation verification + grounding telemetry, #4921). Cache identity * MUST cover every field that shapes the LLM output, or two requests * with the same core fields but different descriptions will share a * cache entry and the second caller gets prose grounded in the first @@ -658,3 +660,316 @@ function containsSubsequence(haystack, needle) { } return false; } + +// ═══════════════════════════════════════════════════════════════════════════ +// Grounding spine (#4921 — ported from scripts/lib/brief-llm.mjs so EVERY +// brief product shares one grounding implementation; brief-llm.mjs +// re-exports these for backcompat). Edge-safe: no node:crypto, no env reads. +// ═══════════════════════════════════════════════════════════════════════════ + + +// Shared delimiter regex for tokenising both story headlines (anchor +// extraction) and synthesis prose (haystack lookup). Same delimiter +// set on both sides keeps the matching contract symmetric. +// +// Unicode quotes (U+2018, U+2019, U+201C, U+201D, U+00B4) are +// included alongside their ASCII counterparts. News headlines from +// Reuters/AP/Guardian use U+2019 for possessives ("China's", +// "Iran's", "DPRK's") and U+201C/U+201D for quoted phrases. Without +// splitting on them, "China's" becomes one token "china’s" that +// a lead saying "China" can never match — a false negative that +// would reject genuinely grounded leads. (PR #3667 review round 2 +// finding #2.) +const GROUNDING_TOKEN_DELIMS = /[\s,.!?;:()'"‘’“”´\\/—–\-[\]{}]+/; + +// Anchor-side stopword list. Story headlines often capitalise +// titles ("President Trump"), generic actors ("Officials confirmed"), +// quasi-adjectives ("Senior commander", "Federal court"), and +// sentence-start filler ("Following the announcement"). Without +// filtering, these enter storyTokens and a hallucinated lead like +// "President Biden announced..." passes the lead-anchor check via +// the shared word "President", then a teaser mentioning a real +// anchor satisfies the combined threshold — the visible top-of- +// email lead stays fabricated. (PR #3667 review round 2 finding #1.) +// +// Scope rule: only words that are commonly capitalised but do NOT +// discriminate a story. Specific entity names (people, places, +// orgs, brands) are NEVER on this list, even when common — "Iran", +// "Trump", "Israel", "EU", "UN" all stay in. "May" is also +// deliberately omitted (Theresa May, May Day, May = month all +// collide on it; safer to keep "may" matchable than to filter it +// and lose a real anchor). +// +// Maintenance heuristic (PR #3667 review round 5 #3): a capitalised +// token of length ≥4 belongs in this set if it appears in >~10% of +// real headlines without discriminating between stories. The cheap +// audit is: dump a week of headlines, tokenise with this same +// extractAnchorTokens function (with stopwords disabled), count +// frequencies, and inspect any token in >50 of ~500 headlines that +// isn't already a known proper noun. The "Prime"/"Chief"/"Cardinal" +// gaps caught on review rounds 2-3 would each have surfaced from +// such a frequency audit. Don't try to enumerate exhaustively up +// front; let production usage drive additions and capture each new +// ride-along bug class as a regression test. +const GROUNDING_ANCHOR_STOPWORDS = new Set([ + // Honorifics / titles + 'president', 'vice', 'senator', 'minister', 'secretary', + 'chairman', 'chairwoman', 'spokesman', 'spokeswoman', + 'director', 'general', 'admiral', 'colonel', 'captain', + 'mayor', 'governor', 'judge', 'justice', 'doctor', + 'professor', 'pope', 'rabbi', 'imam', 'sheikh', 'sultan', + 'emir', 'king', 'queen', 'prince', 'princess', + // Round-3 review additions: bigram-leading titles ("Prime + // Minister", "Chief Justice", "Cardinal Smith") whose first + // word alone passes the cap+length filter and would otherwise + // let a hallucinated "Prime Minister Trudeau announced..." lead + // ride on a "Prime Minister Netanyahu says..." headline via the + // shared "prime" token. PR #3667 review round 3. + 'prime', 'chief', 'premier', 'chancellor', 'speaker', + 'ambassador', 'envoy', 'commissioner', 'attorney', + 'cardinal', 'archbishop', 'monsignor', 'reverend', + 'pastor', 'bishop', 'lord', 'lady', 'dame', + 'congressman', 'congresswoman', 'congressperson', + 'representative', 'delegate', 'baron', 'baroness', + // Generic role plurals / institutional collectives + 'officials', 'officers', 'leaders', 'members', 'people', + 'forces', 'police', 'troops', 'agents', 'authorities', + 'sources', 'rebels', 'militants', 'protesters', 'civilians', + 'residents', 'citizens', 'workers', 'voters', + // Headline qualifiers / quasi-adjectives + 'senior', 'junior', 'former', 'acting', 'deputy', 'assistant', + 'federal', 'national', 'international', 'global', 'regional', + 'central', 'local', 'foreign', 'domestic', 'civil', 'public', + 'private', 'special', 'major', 'armed', + // Sentence-start / common filler + 'after', 'before', 'during', 'while', 'despite', 'following', + 'amid', 'today', 'yesterday', 'tomorrow', 'this', 'these', + 'those', 'when', 'where', 'what', 'which', 'breaking', + // News-headline glue + 'says', 'said', 'told', 'reports', 'analysis', 'opinion', + 'editorial', 'update', 'updates', + // Calendar (May omitted — see scope rule above) + 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', + 'saturday', 'sunday', 'january', 'february', 'march', 'april', + 'june', 'july', 'august', 'september', 'october', 'november', + 'december', +]); + +/** + * Anchor extraction from a story headline: capitalised + length ≥4 + + * NOT in GROUNDING_ANCHOR_STOPWORDS. The capitalisation filter makes + * this a "proper noun" heuristic; the stopword filter strips + * honorifics, role labels, bigram-leading titles, and sentence-start + * filler that would otherwise be shared anchors between any + * "President X..." headline and any "President Y..." hallucinated + * lead. File-level so the closure isn't re-instantiated per + * checkLeadGrounding call (PR #3667 review round 4 P2). + * + * @param {string} s + * @returns {string[]} lowercased anchor tokens + */ +export function extractAnchorTokens(s) { + if (typeof s !== 'string' || s.length === 0) return []; + const out = []; + for (const w of s.split(GROUNDING_TOKEN_DELIMS)) { + if (w.length < 4 || !/^[A-Z]/.test(w)) continue; + const lower = w.toLowerCase(); + if (!GROUNDING_ANCHOR_STOPWORDS.has(lower)) out.push(lower); + } + return out; +} + +/** + * Tokenise synthesis prose into a Set of lowercased words for + * membership lookup. NO capitalisation filter — the synthesis can + * mention the entity in any case (sentence-medial, possessive form, + * etc.) and we still want it to count. File-level for the same + * reason as extractAnchorTokens (PR #3667 review round 4 P2). + * + * @param {string} text + * @returns {Set} + */ +export function groundingTokenSet(text) { + const set = new Set(); + if (typeof text !== 'string' || text.length === 0) return set; + for (const w of text.toLowerCase().split(GROUNDING_TOKEN_DELIMS)) { + if (w.length >= 4) set.add(w); + } + return set; +} + +/** + * Cheap content-grounding check: the canonical lead MUST reference + * proper-noun tokens that actually appear in the input story + * headlines. Without this, the LLM is free to confabulate even with + * shape-valid output — e.g. the 2026-05-12 incident where a Trump- + * era geopolitics pool (Iran/Israel/Sudan/Cuba/Ukraine) shipped a + * "President Biden announced a crypto executive order" lead. Shape + * was valid; content was a complete fabrication the model produced + * from training-data priors instead of grounding. + * + * Two independent grounding requirements (BOTH must pass): + * + * 1. **Lead anchor**: the lead alone must hit ≥1 anchor token. + * Without this, a hallucinated lead can sneak through when the + * threads happen to mention real entities — the visible lead + * stays fabricated even though the combined check passes. + * (Code-review finding on PR #3667 #1.) + * 2. **Combined coverage**: the lead + thread teasers together + * must hit ≥2 anchors (relaxed to 1 when the corpus itself has + * <4 anchor tokens, so single-named-actor briefs aren't + * false-positives). + * + * Matching is **token-set membership** — both sides are split on + * the same delimiter regex and lowercased into Sets. Substring + * matching (the v1 implementation) was rejected on PR #3667 review: + * it accepts unrelated entities like `iran` inside `tirana`, + * `oman` inside `romania`, `india` inside `indiana`. Token-set + * matching avoids that class of false positive cleanly. + * (Code-review finding on PR #3667 #2.) + * + * Length cap of 4 deliberately filters out 2-letter ISO country + * codes (`IR`, `PS`, `US`) and short-form orgs (`UN`, `EU`, `RSF`) + * which are too generic to be discriminating anchors. The check is + * about whether the lead names a SPECIFIC entity — not whether it + * uses any capitalised token at all. + * + * Returns true (grounded, or check-skipped because corpus lacks + * signal / no stories supplied) → accept. Returns false → reject. + * + * @param {{ lead?: string; threads?: Array<{tag?:string;teaser?:string}> }} synthesis + * @param {Array<{ headline?: string }>} stories + * @returns {boolean} + */ +// Default story cap for grounding: mirrors the digest's default +// MAX_STORIES_PER_USER without dragging its env read into this +// edge-safe module — callers with a different cap pass it explicitly. +const DEFAULT_GROUNDING_STORY_CAP = 8; + +export function checkLeadGrounding(synthesis, stories, storyCap = DEFAULT_GROUNDING_STORY_CAP) { + if (!Array.isArray(stories) || stories.length === 0) return true; + + const storyTokens = new Set(); + for (const s of stories.slice(0, storyCap)) { + for (const tok of extractAnchorTokens(s?.headline ?? '')) { + storyTokens.add(tok); + } + } + // Corpus has no proper-noun anchors — can't validate, skip. + // Genuine input (2026-era stories) reliably has >0 such tokens; + // the empty branch is for synthetic / single-headline tests. + // + // Lowercase-headline blind spot (PR #3667 review round 5 #2): + // if a feed ever produces all-lowercase or all-≤3-char headlines, + // every story contributes zero anchors and the gate silently + // skips. Emit a warn so ops can detect the regression — but only + // when stories.length is meaningful (≥3) so the synthetic + // single-headline test corpora don't spam logs. + if (storyTokens.size === 0) { + if (stories.length >= 3) { + console.warn( + `[brief-llm-core] grounding gate skipped: storyTokens empty for stories.length=${stories.length} — likely all-lowercase or <4-char headlines from a feed regression`, + ); + } + return true; + } + + const leadTokens = groundingTokenSet(typeof synthesis?.lead === 'string' ? synthesis.lead : ''); + + // Requirement 1: the lead alone must hit ≥1 anchor. A hallucinated + // lead with grounded teasers would otherwise pass — the user still + // sees the fabricated text at the top of the email. + let leadHasAnchor = false; + for (const tok of leadTokens) { + if (storyTokens.has(tok)) { leadHasAnchor = true; break; } + } + if (!leadHasAnchor) return false; + + // Requirement 2: combined lead + teasers hit ≥threshold anchors. + // Threshold relaxes to 1 when the corpus is sparse so single- + // story briefs don't false-positive. + const combinedTokens = new Set(leadTokens); + for (const t of (Array.isArray(synthesis?.threads) ? synthesis.threads : [])) { + for (const w of groundingTokenSet(typeof t?.teaser === 'string' ? t.teaser : '')) { + combinedTokens.add(w); + } + } + const threshold = storyTokens.size >= 4 ? 2 : 1; + let combinedHits = 0; + for (const tok of storyTokens) { + if (combinedTokens.has(tok)) { + combinedHits++; + if (combinedHits >= threshold) return true; + } + } + return false; +} + +/** + * Lead ↔ single-story coherence check (F4). Returns true iff `lead` + * shares ≥1 proper-noun anchor with `headline`. Reuses the same + * anchor machinery as `checkLeadGrounding` (capitalised, length ≥4, + * stopword-filtered headline anchors; token-set membership against + * the lead) but with a FIXED threshold of 1 — coherence asks only + * "is the lead about the same story?", not "how well-grounded is it?". + * + * `checkLeadGrounding` itself is the wrong fit here: scoped to one + * story, a single headline can carry ≥4 anchor tokens, which trips + * its `size >= 4 ? 2 : 1` threshold up to 2 — too strict for + * coherence, where a lead legitimately about card #1 may name only + * one of its entities. + * + * Used by the cron's lead/card-#1 coherence telemetry + * (`composeAndStoreBriefForUser`) — see plan + * docs/plans/2026-05-14-001-…-plan.md (F4, Phase 4). + * + * @param {string} lead — the canonical synthesis lead + * @param {string} headline — the rendered first card's headline + * @returns {boolean} true = coherent (or check-skipped); false = the + * lead names none of the headline's proper-noun anchors + */ +export function leadGroundsAgainstStory(lead, headline) { + const anchors = new Set(extractAnchorTokens(typeof headline === 'string' ? headline : '')); + // No proper-noun anchors in the headline → cannot judge coherence, + // skip (same "degenerate corpus → accept" stance as checkLeadGrounding). + if (anchors.size === 0) return true; + const leadTokens = groundingTokenSet(typeof lead === 'string' ? lead : ''); + for (const tok of anchors) { + if (leadTokens.has(tok)) return true; + } + return false; +} + + +/** + * #4921: mechanical citation verification. Every bracket citation [n] + * in LLM brief prose must map to a real entry in the grounding source + * list (1..sourceCount). Out-of-range markers are invented — strip them + * rather than rendering a dead reference. Returns the cleaned text and + * the count of stripped markers (callers log/telemeter when > 0). + * + * Pure string operation; deliberately does NOT try to verify the CLAIM + * against the source — that is the grounding gates' job. This closes + * the cheaper hole: "[9]" shipped against a 6-source list. + * + * @param {string} text + * @param {number} sourceCount + * @returns {{ text: string; stripped: number }} + */ +export function verifyCitationIndexes(text, sourceCount) { + if (typeof text !== 'string' || text.length === 0) { + return { text: typeof text === 'string' ? text : '', stripped: 0 }; + } + const max = Number.isFinite(sourceCount) && sourceCount > 0 ? Math.floor(sourceCount) : 0; + let stripped = 0; + // 1-3 digits: [123] must not sail through unverified (review finding); + // 4+ digit brackets ([2026]) are treated as prose, not citations. + const cleaned = text.replace(/\s*\[(\d{1,3})\]/g, (full, numStr) => { + const n = Number.parseInt(numStr, 10); + if (n >= 1 && n <= max) return full; + stripped++; + return ''; + }); + return { text: cleaned, stripped }; +} + diff --git a/src/components/InsightsPanel.ts b/src/components/InsightsPanel.ts index 377b263014..43234a4c08 100644 --- a/src/components/InsightsPanel.ts +++ b/src/components/InsightsPanel.ts @@ -12,6 +12,7 @@ import { getCachedPosture } from '@/services/cached-theater-posture'; import { isMobileDevice } from '@/utils'; import { escapeHtml, sanitizeUrl, unsafeRawHtml } from '@/utils/sanitize'; import { collectBriefSources, normalizeCachedBriefSources, renderBriefSourcesFooter, type BriefSource } from '@/utils/brief-sources'; +import { formatIntelBrief } from '@/utils/format-intel-brief'; import { SITE_VARIANT } from '@/config'; import { deletePersistentCache, getPersistentCache, setPersistentCache } from '@/services/persistent-cache'; import { t } from '@/services/i18n'; @@ -513,11 +514,16 @@ export class InsightsPanel extends Panel { insights: ServerInsights, sentiments: Array<{ label: string; score: number }> | null, ): void { + // #4928 external review: the synthesis cites up to 8 stories — a + // 6-source cap orphaned [7]/[8]. Cap to the payload's own citation + // index space (bounded at 12 defensively). const worldBriefSources = collectBriefSources( insights.worldBriefSources ?? [], - 6, + Math.min(12, Math.max(6, insights.worldBriefSources?.length ?? 6)), ); - const briefHtml = insights.worldBrief ? this.renderWorldBrief(insights.worldBrief, worldBriefSources) : ''; + const briefHtml = insights.worldBrief + ? this.renderWorldBrief(insights.worldBrief, worldBriefSources, this.renderBriefExtras(insights)) + : ''; const focalPointsHtml = this.renderFocalPoints(); const convergenceHtml = this.renderConvergenceZones(); const sentimentOverview = this.renderSentimentOverview(sentiments); @@ -613,7 +619,36 @@ export class InsightsPanel extends Panel { `; } - private renderWorldBrief(brief: string, sources: BriefSource[] = []): string { + /** #4921: cited per-story lines + staleness footer for the World Brief. */ + private renderBriefExtras(insights: ServerInsights): string { + const lines = Array.isArray(insights.briefStoryLines) ? insights.briefStoryLines : []; + const sources = insights.worldBriefSources ?? []; + const linesHtml = lines.length > 0 + ? `
    ${lines + .map((line) => `
  1. ${formatIntelBrief(line.text, { sources }) + .replace(/^
    /, '') + .replace(/<\/div>$/, '') + .replace(/^

    /, '') + .replace(/<\/p>$/, '')}

    2. `) + .join('')}
` + : ''; + let footer = ''; + const generatedMs = new Date(insights.generatedAt).getTime(); + const newestMs = insights.sourceAgeRange?.newestMs; + // Pre-rollout payloads lack sourceAgeRange — omit the footer rather + // than rendering a literal "?h old" (#4928 external review P3). + if (Number.isFinite(generatedMs) && Number.isFinite(newestMs)) { + const agoMin = Math.max(0, Math.round((Date.now() - generatedMs) / 60000)); + const newestAgeH = Math.max(0, Math.round((Date.now() - (newestMs as number)) / 3600000 * 10) / 10); + footer = `
${t('components.insights.briefFreshness', { + minutes: String(agoMin), + hours: String(newestAgeH), + })}
`; + } + return linesHtml + footer; + } + + private renderWorldBrief(brief: string, sources: BriefSource[] = [], extrasHtml = ''): string { const heading = SITE_VARIANT === 'tech' ? `🚀 ${t('components.insights.briefTech')}` : SITE_VARIANT === 'commodity' ? `⛏️ ${t('components.insights.briefCommodity')}` @@ -623,7 +658,8 @@ export class InsightsPanel extends Panel {
${heading}
${escapeHtml(brief)}
- ${renderBriefSourcesFooter(sources, { className: 'insights-brief-sources' })} + ${extrasHtml} + ${renderBriefSourcesFooter(sources, { className: 'insights-brief-sources', maxSources: Math.max(6, sources.length) })}
`; } diff --git a/src/locales/ar.json b/src/locales/ar.json index 42cc7a67ce..ff6a7de385 100644 --- a/src/locales/ar.json +++ b/src/locales/ar.json @@ -1265,7 +1265,8 @@ "sources_few": "{{count}} مصادر", "sources_many": "{{count}} مصدر", "provenanceTitle": "أصل التغطية: عدد القصص المأخوذة والمصادر المميزة التي تم اختيار هذا الملخص منها", - "compiledFrom": "مجمّع من {{stories}} قصة عبر {{sources}} مصادر" + "compiledFrom": "مجمّع من {{stories}} قصة عبر {{sources}} مصادر", + "briefFreshness": "تم الإنشاء منذ {{minutes}}د · أحدث مصدر عمره {{hours}}س" }, "cascade": { "filters": { diff --git a/src/locales/bg.json b/src/locales/bg.json index 0b6b1000f8..71ece886aa 100644 --- a/src/locales/bg.json +++ b/src/locales/bg.json @@ -1304,7 +1304,8 @@ "signalTypesEvents": "{{types}} типа сигнали • {{events}} събития", "newsSignals": "{{news}} новини • {{signals}} сигнали", "provenanceTitle": "Произход на обхвата: колко включени истории и отделни източници е избрано от този преглед", - "compiledFrom": "Събрано от {{stories}} истории в {{sources}} източници" + "compiledFrom": "Събрано от {{stories}} истории в {{sources}} източници", + "briefFreshness": "Генериран преди {{minutes}}м · най-нов източник {{hours}}ч стар" }, "settings": { "exportSettings": "Експорт на настройки", diff --git a/src/locales/cs.json b/src/locales/cs.json index 337e345c90..3ccd3f0e73 100644 --- a/src/locales/cs.json +++ b/src/locales/cs.json @@ -1294,7 +1294,8 @@ "sources_few": "{{count}} zdroje", "sources_many": "{{count}} zdrojů", "provenanceTitle": "Původ pokrytí: počet příjatých příběhů a odlišných zdrojů, ze kterých byl tento přehled vybrán", - "compiledFrom": "Sestaveno z {{stories}} příběhů z {{sources}} zdrojů" + "compiledFrom": "Sestaveno z {{stories}} příběhů z {{sources}} zdrojů", + "briefFreshness": "Vygenerováno před {{minutes}}m · nejnovější zdroj starý {{hours}}h" }, "cascade": { "filters": { diff --git a/src/locales/de.json b/src/locales/de.json index 3f3971e5b5..bc89fdb085 100644 --- a/src/locales/de.json +++ b/src/locales/de.json @@ -1604,7 +1604,8 @@ "signalTypesEvents": "{{types}} Signaltypen • {{events}} Ereignisse", "newsSignals": "{{news}} Nachrichten • {{signals}} Signale", "provenanceTitle": "Abdeckungsherkunft: Anzahl der erfassten Stories und unterschiedlichen Quellen, aus denen dieser Brief ausgewählt wurde", - "compiledFrom": "Zusammengestellt aus {{stories}} Geschichten über {{sources}} Quellen" + "compiledFrom": "Zusammengestellt aus {{stories}} Geschichten über {{sources}} Quellen", + "briefFreshness": "Generiert vor {{minutes}}m · neueste Quelle {{hours}}h alt" }, "etfFlows": { "unavailable": "ETF-Daten vorübergehend nicht verfügbar", diff --git a/src/locales/el.json b/src/locales/el.json index 8a65b8edc8..0504de4dfc 100644 --- a/src/locales/el.json +++ b/src/locales/el.json @@ -1304,7 +1304,8 @@ "signalTypesEvents": "{{types}} τύποι σήματος • {{events}} συμβάντα", "newsSignals": "{{news}} ειδήσεις • {{signals}} σήματα", "provenanceTitle": "Προέλευση κάλυψης: πόσες καταχωρημένες ιστορίες και διακριτές πηγές επιλέχθησαν για αυτή τη σύνοψη", - "compiledFrom": "Συντάχθηκε από {{stories}} ιστορίες σε {{sources}} πηγές" + "compiledFrom": "Συντάχθηκε από {{stories}} ιστορίες σε {{sources}} πηγές", + "briefFreshness": "Δημιουργήθηκε πριν {{minutes}}λ · νεότερη πηγή {{hours}}ώ παλιά" }, "settings": { "exportSettings": "Εξαγωγή ρυθμίσεων", diff --git a/src/locales/en.json b/src/locales/en.json index 65e3c6e45f..97504159c1 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -1444,7 +1444,8 @@ "signalTypesEvents": "{{types}} signal types • {{events}} events", "newsSignals": "{{news}} news • {{signals}} signals", "compiledFrom": "Compiled from {{stories}} stories across {{sources}} sources", - "provenanceTitle": "Coverage provenance: how many ingested stories and distinct sources this brief was selected from" + "provenanceTitle": "Coverage provenance: how many ingested stories and distinct sources this brief was selected from", + "briefFreshness": "Generated {{minutes}}m ago · newest source {{hours}}h old" }, "settings": { "exportSettings": "Export Settings", diff --git a/src/locales/es.json b/src/locales/es.json index f1b0d13055..9d68823850 100644 --- a/src/locales/es.json +++ b/src/locales/es.json @@ -1605,7 +1605,8 @@ "newsSignals": "{{news}} noticias • {{signals}} señales", "sources_many": "{{count}} fuentes", "provenanceTitle": "Procedencia de cobertura: cuántas historias ingeridas y fuentes distintas se seleccionaron para este resumen", - "compiledFrom": "Compilado de {{stories}} historias en {{sources}} fuentes" + "compiledFrom": "Compilado de {{stories}} historias en {{sources}} fuentes", + "briefFreshness": "Generado hace {{minutes}}m · fuente más reciente con {{hours}}h de antigüedad" }, "etfFlows": { "unavailable": "Datos ETF temporalmente no disponibles", diff --git a/src/locales/fa.json b/src/locales/fa.json index cfe0dc68e3..8c4dda48af 100644 --- a/src/locales/fa.json +++ b/src/locales/fa.json @@ -1444,7 +1444,8 @@ "signalTypesEvents": "{{types}} signal types • {{events}} events", "newsSignals": "{{news}} news • {{signals}} signals", "compiledFrom": "گردآوری‌شده از {{stories}} خبر از {{sources}} منبع", - "provenanceTitle": "شفافیت پوشش: این گزیده از چه تعداد خبر دریافتی و چند منبع مجزا انتخاب شده است" + "provenanceTitle": "شفافیت پوشش: این گزیده از چه تعداد خبر دریافتی و چند منبع مجزا انتخاب شده است", + "briefFreshness": "تولید شده {{minutes}}m پیش · جدیدترین منبع {{hours}}h قدیمی" }, "settings": { "exportSettings": "Export Settings", diff --git a/src/locales/fr.json b/src/locales/fr.json index aa3f8575bf..f5cdc52a5a 100644 --- a/src/locales/fr.json +++ b/src/locales/fr.json @@ -1262,7 +1262,8 @@ "newsSignals": "{{news}} actualités • {{signals}} signaux", "sources_many": "{{count}} sources", "provenanceTitle": "Provenance de la couverture : nombre d'articles ingérés et de sources distinctes à partir desquels ce résumé a été sélectionné", - "compiledFrom": "Compilé à partir de {{stories}} articles provenant de {{sources}} sources" + "compiledFrom": "Compilé à partir de {{stories}} articles provenant de {{sources}} sources", + "briefFreshness": "Généré il y a {{minutes}}m · source la plus récente datant de {{hours}}h" }, "settings": { "exportSettings": "Exporter les paramètres", diff --git a/src/locales/hi.json b/src/locales/hi.json index f4fdadebf3..6dda5196d1 100644 --- a/src/locales/hi.json +++ b/src/locales/hi.json @@ -1444,7 +1444,8 @@ "signalTypesEvents": "{{types}} संकेत प्रकार • {{events}} घटनाएं", "newsSignals": "{{news}} खबरें • {{signals}} संकेत", "provenanceTitle": "कवरेज प्रोवेनेंस: इस ब्रीफ को कितनी अंतर्ग्रहीत कहानियों और विशिष्ट स्रोतों से चुना गया था", - "compiledFrom": "{{stories}} कहानियों और {{sources}} स्रोतों से संकलित" + "compiledFrom": "{{stories}} कहानियों और {{sources}} स्रोतों से संकलित", + "briefFreshness": "{{minutes}}m पहले जनरेट किया गया · नवीनतम स्रोत {{hours}}h पुराना" }, "settings": { "exportSettings": "सेटिंग्स निर्यात करें", diff --git a/src/locales/hr.json b/src/locales/hr.json index bea39806ae..708b965ffd 100644 --- a/src/locales/hr.json +++ b/src/locales/hr.json @@ -1579,7 +1579,8 @@ "newsSignals": "{{news}} vijesti • {{signals}} signala", "sources_few": "{{count}} izvora", "provenanceTitle": "Pokrivanje izvora: koliko je unesenih članaka i različitih izvora odabrano za ovaj sažetak", - "compiledFrom": "Sastavljeno od {{stories}} priča iz {{sources}} izvora" + "compiledFrom": "Sastavljeno od {{stories}} priča iz {{sources}} izvora", + "briefFreshness": "Generirano prije {{minutes}}m · najnoviji izvor star {{hours}}h" }, "settings": { "dataManagementLabel": "Upravljanje podacima", diff --git a/src/locales/hu.json b/src/locales/hu.json index 0eb23caaeb..cf370ddbd7 100644 --- a/src/locales/hu.json +++ b/src/locales/hu.json @@ -1444,7 +1444,8 @@ "signalTypesEvents": "{{types}} szignáltípus • {{events}} esemény", "newsSignals": "{{news}} hír • {{signals}} szignál", "provenanceTitle": "Lefedettség eredete: hány feldolgozott történetből és különálló forrásból választotta ki ezt az összefoglalót", - "compiledFrom": "{{stories}} történetből és {{sources}} forrásból összeállítva" + "compiledFrom": "{{stories}} történetből és {{sources}} forrásból összeállítva", + "briefFreshness": "Generálva {{minutes}}m idővel ezelőtt · legfrissebb forrás {{hours}}ó régi" }, "settings": { "exportSettings": "Beállítások exportálása", diff --git a/src/locales/it.json b/src/locales/it.json index ceb0ca3c3a..5a22fd054d 100644 --- a/src/locales/it.json +++ b/src/locales/it.json @@ -1605,7 +1605,8 @@ "newsSignals": "{{news}} notizie • {{signals}} segnali", "sources_many": "{{count}} fonti", "provenanceTitle": "Provenienza della copertura: quante storie acquisite e quante fonti distinte sono state utilizzate per selezionare questo briefing", - "compiledFrom": "Compilato da {{stories}} storie provenienti da {{sources}} fonti" + "compiledFrom": "Compilato da {{stories}} storie provenienti da {{sources}} fonti", + "briefFreshness": "Generato {{minutes}}m fa · fonte più recente {{hours}}h fa" }, "etfFlows": { "unavailable": "Dati ETF temporaneamente non disponibili", diff --git a/src/locales/ja.json b/src/locales/ja.json index c127468e10..c348f2638e 100644 --- a/src/locales/ja.json +++ b/src/locales/ja.json @@ -1304,7 +1304,8 @@ "signalTypesEvents": "{{types}} シグナルタイプ • {{events}} イベント", "newsSignals": "{{news}} ニュース • {{signals}} シグナル", "provenanceTitle": "カバレッジプロヴェナンス: このブリーフが選択された取り込まれたストーリーと個別ソースの数", - "compiledFrom": "{{stories}}件のストーリーから{{sources}}個のソースをコンパイル" + "compiledFrom": "{{stories}}件のストーリーから{{sources}}個のソースをコンパイル", + "briefFreshness": "{{minutes}}m前に生成 · 最新ソース{{hours}}h前" }, "settings": { "exportSettings": "設定をエクスポート", diff --git a/src/locales/ko.json b/src/locales/ko.json index 5fe8055445..671b3e81c0 100644 --- a/src/locales/ko.json +++ b/src/locales/ko.json @@ -1304,7 +1304,8 @@ "signalTypesEvents": "{{types}}개 신호 유형 • {{events}}개 이벤트", "newsSignals": "{{news}}개 뉴스 • {{signals}}개 신호", "provenanceTitle": "커버리지 출처: 이 브리프가 선택된 수집 스토리 및 고유 소스의 개수", - "compiledFrom": "{{stories}}개의 스토리와 {{sources}}개의 소스에서 컴파일됨" + "compiledFrom": "{{stories}}개의 스토리와 {{sources}}개의 소스에서 컴파일됨", + "briefFreshness": "{{minutes}}분 전 생성됨 · 최신 소스 {{hours}}시간 전" }, "settings": { "exportSettings": "설정 내보내기", diff --git a/src/locales/nl.json b/src/locales/nl.json index b62959ecd5..4b31101950 100644 --- a/src/locales/nl.json +++ b/src/locales/nl.json @@ -1276,7 +1276,8 @@ "signalTypesEvents": "{{types}} signaaltypen • {{events}} gebeurtenissen", "newsSignals": "{{news}} nieuws • {{signals}} signalen", "provenanceTitle": "Dekking herkomst: hoeveel opgenomen verhalen en verschillende bronnen deze samenvatting is geselecteerd uit", - "compiledFrom": "Samengesteld uit {{stories}} verhalen uit {{sources}} bronnen" + "compiledFrom": "Samengesteld uit {{stories}} verhalen uit {{sources}} bronnen", + "briefFreshness": "Gegenereerd {{minutes}}m geleden · nieuwste bron {{hours}}h oud" }, "techHubs": { "infoTooltip": "Tech Hub-activiteit
Toont tech-hubs met de meeste nieuwsactiviteit.

Activiteitsniveaus:
• Hoog — Breaking news of 50+ score
• Verhoogd — Score 20-49
• Laag — Score lager dan 20

Klik op een hub om naar de locatie ervan te zoomen." diff --git a/src/locales/pl.json b/src/locales/pl.json index 6e1f63b596..5c18718ce2 100644 --- a/src/locales/pl.json +++ b/src/locales/pl.json @@ -1542,7 +1542,8 @@ "sources_few": "{{count}} źródła", "sources_many": "{{count}} źródeł", "provenanceTitle": "Pochodzenie zasięgu: ile pozyskanych artykułów i odrębnych źródeł wybrano dla tego briefu", - "compiledFrom": "Zebrane z {{stories}} artykułów z {{sources}} źródeł" + "compiledFrom": "Zebrane z {{stories}} artykułów z {{sources}} źródeł", + "briefFreshness": "Wygenerowano {{minutes}}m temu · najnowsze źródło sprzed {{hours}}h" }, "etfFlows": { "unavailable": "Dane ETF tymczasowo niedostępne", diff --git a/src/locales/pt.json b/src/locales/pt.json index 9018f5dfa6..60d333d46e 100644 --- a/src/locales/pt.json +++ b/src/locales/pt.json @@ -1277,7 +1277,8 @@ "newsSignals": "{{news}} notícias • {{signals}} sinais", "sources_many": "{{count}} fontes", "provenanceTitle": "Proveniência de cobertura: quantas histórias ingeridas e fontes distintas foram selecionadas para este resumo", - "compiledFrom": "Compilado de {{stories}} histórias em {{sources}} fontes" + "compiledFrom": "Compilado de {{stories}} histórias em {{sources}} fontes", + "briefFreshness": "Gerado há {{minutes}}m · fonte mais recente com {{hours}}h" }, "techHubs": { "infoTooltip": "Atividade do Tech Hub
Mostra os hubs de tecnologia com mais atividade de notícias.

Níveis de atividade:
• Alto — Notícias de última hora ou pontuação acima de 50
• Elevado — Pontuação 20-49
• Baixo — Pontuação abaixo de 20

Clique em um hub para ampliar sua localização." diff --git a/src/locales/ro.json b/src/locales/ro.json index 5668412d63..3cba45a483 100644 --- a/src/locales/ro.json +++ b/src/locales/ro.json @@ -1305,7 +1305,8 @@ "newsSignals": "{{news}} știri • {{signals}} semnale", "sources_few": "{{count}} surse", "provenanceTitle": "Proveniență acoperire: câte articole ingerate și surse distincte din care a fost selectat acest rezumat", - "compiledFrom": "Compilat din {{stories}} povești din {{sources}} surse" + "compiledFrom": "Compilat din {{stories}} povești din {{sources}} surse", + "briefFreshness": "Generat {{minutes}}m în urmă · cea mai nouă sursă {{hours}}h vechime" }, "settings": { "exportSettings": "Exportare setări", diff --git a/src/locales/ru.json b/src/locales/ru.json index 2cba74dd16..7c385b1904 100644 --- a/src/locales/ru.json +++ b/src/locales/ru.json @@ -1306,7 +1306,8 @@ "sources_few": "{{count}} источника", "sources_many": "{{count}} источников", "provenanceTitle": "Происхождение охвата: количество полученных историй и отдельных источников, из которых была выбрана эта сводка", - "compiledFrom": "Собрано из {{stories}} историй из {{sources}} источников" + "compiledFrom": "Собрано из {{stories}} историй из {{sources}} источников", + "briefFreshness": "Сгенерировано {{minutes}}м назад · последний источник {{hours}}ч старый" }, "settings": { "exportSettings": "Экспорт настроек", diff --git a/src/locales/sv.json b/src/locales/sv.json index fb9d4677fe..33d2ddb79c 100644 --- a/src/locales/sv.json +++ b/src/locales/sv.json @@ -1276,7 +1276,8 @@ "signalTypesEvents": "{{types}} signaltyper • {{events}} händelser", "newsSignals": "{{news}} nyheter • {{signals}} signaler", "provenanceTitle": "Täckningshärkomst: hur många inmatade historier och unika källor denna kortfattade sammanfattning valdes från", - "compiledFrom": "Kompilerat från {{stories}} berättelser från {{sources}} källor" + "compiledFrom": "Kompilerat från {{stories}} berättelser från {{sources}} källor", + "briefFreshness": "Genererad {{minutes}}m sedan · senaste källa {{hours}}h gammal" }, "techHubs": { "infoTooltip": "Teknisk hubbaktivitet
Visar tekniska nav med mest nyhetsaktivitet.

Aktivitetsnivåer:
• Hög — Nyheter från 50+ PH_• 50+ stil: {{elevatedColor}}\">Höjd — Poäng 20-49
• Låg — Poäng under 20

Klicka på ett nav för att zooma till dess plats." diff --git a/src/locales/th.json b/src/locales/th.json index 2d216cbc78..4f55a0a114 100644 --- a/src/locales/th.json +++ b/src/locales/th.json @@ -1304,7 +1304,8 @@ "signalTypesEvents": "{{types}} ประเภทสัญญาณ • {{events}} เหตุการณ์", "newsSignals": "{{news}} ข่าว • {{signals}} สัญญาณ", "provenanceTitle": "ความมาของการครอบคลุม: จำนวนเรื่องข่าวที่นำเข้าและแหล่งข่าวที่แตกต่างกันที่ใช้สำหรับสรุปนี้", - "compiledFrom": "รวบรวมจาก {{stories}} เรื่องราวจาก {{sources}} แหล่งที่มา" + "compiledFrom": "รวบรวมจาก {{stories}} เรื่องราวจาก {{sources}} แหล่งที่มา", + "briefFreshness": "สร้างเมื่อ {{minutes}}m ที่แล้ว · แหล่งข้อมูลล่าสุด {{hours}}h เก่า" }, "settings": { "exportSettings": "ส่งออกการตั้งค่า", diff --git a/src/locales/tr.json b/src/locales/tr.json index 1e586d01d0..02cca0e3f2 100644 --- a/src/locales/tr.json +++ b/src/locales/tr.json @@ -1304,7 +1304,8 @@ "signalTypesEvents": "{{types}} sinyal türü • {{events}} olay", "newsSignals": "{{news}} haber • {{signals}} sinyal", "compiledFrom": "{{stories}} hikaye ve {{sources}} kaynaktan derlenmiştir", - "provenanceTitle": "Kapsam kaynağı: bu özet için kaç alınan hikaye ve farklı kaynaktan seçildiği" + "provenanceTitle": "Kapsam kaynağı: bu özet için kaç alınan hikaye ve farklı kaynaktan seçildiği", + "briefFreshness": "{{minutes}}m önce oluşturuldu · en yeni kaynak {{hours}}s eski" }, "settings": { "exportSettings": "Ayarları Dışa Aktar", diff --git a/src/locales/vi.json b/src/locales/vi.json index 8fce2e3d09..2bfffee0a0 100644 --- a/src/locales/vi.json +++ b/src/locales/vi.json @@ -1304,7 +1304,8 @@ "signalTypesEvents": "{{types}} loại tín hiệu • {{events}} sự kiện", "newsSignals": "{{news}} tin tức • {{signals}} tín hiệu", "provenanceTitle": "Nguồn gốc bài viết: số lượng bài viết được nhập và các nguồn riêng biệt mà bài viết này được chọn từ đó", - "compiledFrom": "Được biên soạn từ {{stories}} câu chuyện trên {{sources}} nguồn" + "compiledFrom": "Được biên soạn từ {{stories}} câu chuyện trên {{sources}} nguồn", + "briefFreshness": "Được tạo {{minutes}}m trước · nguồn mới nhất {{hours}}h tuổi" }, "settings": { "exportSettings": "Xuất cài đặt", diff --git a/src/locales/zh.json b/src/locales/zh.json index e5d0c8d7c9..73db447b21 100644 --- a/src/locales/zh.json +++ b/src/locales/zh.json @@ -1304,7 +1304,8 @@ "signalTypesEvents": "{{types}} 种信号类型 • {{events}} 个事件", "newsSignals": "{{news}} 条新闻 • {{signals}} 个信号", "compiledFrom": "编译自 {{stories}} 条故事和 {{sources}} 个来源", - "provenanceTitle": "覆盖来源：此摘要选自多少条摄取的故事和不同的来源" + "provenanceTitle": "覆盖来源：此摘要选自多少条摄取的故事和不同的来源", + "briefFreshness": "生成于 {{minutes}}m 前 · 最新来源 {{hours}}h 前" }, "settings": { "exportSettings": "导出设置", diff --git a/src/services/insights-loader.ts b/src/services/insights-loader.ts index 99b9537bd8..c4d5c51686 100644 --- a/src/services/insights-loader.ts +++ b/src/services/insights-loader.ts @@ -24,6 +24,11 @@ export interface ServerBriefSource { export interface ServerInsights { worldBrief: string; + /** #4921: one cited line per top story from the synthesis call — + * absent on pre-rollout payloads and single-headline (L2) briefs. */ + briefStoryLines?: Array<{ n: number; text: string }>; + /** #4921: age window of the source material behind this brief. */ + sourceAgeRange?: { newestMs: number; oldestMs: number } | null; worldBriefSources?: ServerBriefSource[]; briefProvider: string; status: 'ok' | 'degraded'; diff --git a/src/styles/main.css b/src/styles/main.css index a1610d044b..92ca256d79 100644 --- a/src/styles/main.css +++ b/src/styles/main.css @@ -17948,6 +17948,23 @@ a.prediction-link:hover { flex: 1; } +/* #4921 cited per-story brief lines + freshness footer */ +.insights-brief-lines { + margin: 8px 0 4px; + padding-left: 18px; + font-size: 11px; + line-height: 1.5; + color: var(--text); +} +.insights-brief-lines li { + margin-bottom: 3px; +} +.insights-brief-freshness { + font-size: 9px; + color: var(--text-dim); + margin-top: 4px; +} + /* #4920 coverage provenance stamp under the insights stats */ .insights-provenance { font-size: 10px; diff --git a/tests/brief-contract.test.mjs b/tests/brief-contract.test.mjs new file mode 100644 index 0000000000..e9139ae098 --- /dev/null +++ b/tests/brief-contract.test.mjs @@ -0,0 +1,361 @@ +// #4921: the brief contract — top-8 synthesis prompts/parser, mechanical +// citation verification, the grounding spine port, and wiring assertions. + +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { + synthesisSystemPrompt, + synthesisUserPrompt, + parseBriefSynthesis, +} from '../scripts/_insights-brief.mjs'; +import { + verifyCitationIndexes, + checkLeadGrounding, + leadGroundsAgainstStory, + extractAnchorTokens, +} from '../shared/brief-llm-core.js'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const readSrc = (rel) => readFileSync(resolve(root, rel), 'utf-8'); + +const STORIES = [ + { primaryTitle: 'Iran threatens to close Strait of Hormuz', primarySource: 'Reuters', sources: ['Reuters', 'BBC'] }, + { primaryTitle: 'Turkey hikes interest rates to 50%', primarySource: 'Bloomberg', sources: ['Bloomberg'] }, + { primaryTitle: 'Magnitude 6.8 earthquake strikes northern Chile', primarySource: 'AP', sources: ['AP', 'AFP', 'CNN'] }, +]; + +describe('synthesis prompts (#4921)', () => { + it('system prompt demands JSON, per-story lines, citations, and no invention', () => { + const prompt = synthesisSystemPrompt('2026-07-06'); + assert.match(prompt, /JSON ONLY/); + assert.match(prompt, /one entry per numbered story/); + assert.match(prompt, /\[n\]|\[1\]/); + assert.match(prompt, /Do not invent proper nouns/); + assert.match(prompt, /ONLY facts present/); + }); + + it('user prompt numbers every story with source counts', () => { + const prompt = synthesisUserPrompt(STORIES); + assert.match(prompt, /1\. Iran threatens to close Strait of Hormuz \(Reuters, 2 sources\)/); + assert.match(prompt, /2\. Turkey hikes interest rates to 50% \(Bloomberg, 1 source\)/); + assert.match(prompt, /3\. Magnitude 6\.8 earthquake/); + }); +}); + +describe('parseBriefSynthesis (#4921)', () => { + const VALID = JSON.stringify({ + lead: 'Iran escalates around Hormuz [1] while Turkey moves rates sharply higher [2] and Chile digs out from a major quake [3].', + lines: [ + { n: 1, text: 'Iran threatens to close the Strait of Hormuz [1].' }, + { n: 2, text: 'Turkey raises interest rates to 50% [2].' }, + { n: 3, text: 'A 6.8-magnitude earthquake strikes northern Chile [3].' }, + ], + }); + + it('parses clean JSON', () => { + const out = parseBriefSynthesis(VALID, 3); + assert.ok(out); + assert.equal(out.lines.length, 3); + assert.match(out.lead, /Hormuz/); + }); + + it('strips markdown fences (groq/Gemini wrap)', () => { + const out = parseBriefSynthesis('```json\n' + VALID + '\n```', 3); + assert.ok(out, 'fenced JSON must parse'); + }); + + it('tolerates prose around the JSON object', () => { + const out = parseBriefSynthesis('Here is the brief:\n' + VALID + '\nHope that helps!', 3); + assert.ok(out); + }); + + it('rejects out-of-range and duplicate line indexes, keeps valid ones', () => { + const messy = JSON.stringify({ + lead: 'Iran and Turkey dominate the day with Hormuz tension and a sharp rate move [1][2].', + lines: [ + { n: 0, text: 'Out of range line that should be discarded entirely.' }, + { n: 1, text: 'Iran threatens to close the Strait of Hormuz [1].' }, + { n: 1, text: 'Duplicate index must not override the first entry.' }, + { n: 9, text: 'Also out of range for a 3-story brief input.' }, + { n: 2, text: 'Turkey raises interest rates to 50% [2].' }, + ], + }); + const out = parseBriefSynthesis(messy, 3); + assert.ok(out); + assert.deepEqual(out.lines.map((l) => l.n), [1, 2]); + assert.match(out.lines[0].text, /Hormuz/); + }); + + it('returns null when fewer than half the stories have usable lines', () => { + const thin = JSON.stringify({ + lead: 'A lead that is long enough to pass the basic length validation gate here.', + lines: [{ n: 1, text: 'Only one usable line for an eight-story brief input.' }], + }); + assert.equal(parseBriefSynthesis(thin, 8), null); + }); + + it('returns null on garbage and on missing lead', () => { + assert.equal(parseBriefSynthesis('not json at all', 3), null); + assert.equal(parseBriefSynthesis(JSON.stringify({ lines: [] }), 3), null); + }); +}); + +describe('verifyCitationIndexes (#4921)', () => { + it('keeps in-range citations, strips invented ones', () => { + const { text, stripped } = verifyCitationIndexes('Tension rises [1] as markets react [7] to the move [2].', 3); + assert.equal(stripped, 1); + assert.match(text, /\[1\]/); + assert.match(text, /\[2\]/); + assert.doesNotMatch(text, /\[7\]/); + }); + + it('zero sources strips every citation', () => { + const { text, stripped } = verifyCitationIndexes('Claim [1] and claim [2].', 0); + assert.equal(stripped, 2); + assert.doesNotMatch(text, /\[\d\]/); + }); + + it('non-string input degrades safely', () => { + assert.deepEqual(verifyCitationIndexes(null, 3), { text: '', stripped: 0 }); + }); +}); + +describe('grounding spine port (#4921)', () => { + it('core exports work standalone with the cap parameter', () => { + const stories = STORIES.map((s) => ({ headline: s.primaryTitle })); + assert.equal(checkLeadGrounding({ lead: 'Iran moves on Hormuz as Turkey acts.' }, stories, 8), true); + assert.equal( + checkLeadGrounding({ lead: 'President Biden announced a crypto executive order today.' }, stories, 8), + false, + 'fabricated lead must fail grounding', + ); + assert.equal(leadGroundsAgainstStory('Iran escalates', 'Iran threatens to close Strait of Hormuz'), true); + assert.ok(extractAnchorTokens('Iran threatens Hormuz closure').includes('iran')); + }); + + it('brief-llm.mjs re-exports the core implementation (no drift possible)', async () => { + const lib = await import('../scripts/lib/brief-llm.mjs'); + const core = await import('../shared/brief-llm-core.js'); + assert.equal(lib.checkLeadGrounding, core.checkLeadGrounding, 'must be the SAME function object'); + assert.equal(lib.leadGroundsAgainstStory, core.leadGroundsAgainstStory); + }); +}); + +describe('brief-contract wiring (source-textual)', () => { + it('seed-insights runs the synthesis path through the pure composer with enforce-by-default', () => { + const src = readSrc('scripts/seed-insights.mjs'); + assert.match(src, /synthesisSystemPrompt/); + assert.match(src, /composeSynthesizedBrief\(synthesisResult\.text, topStories, \{/); + assert.match(src, /validatorMode: BRIEF_VALIDATOR_MODE/); + assert.match(src, /=== 'shadow' \? 'shadow' : 'enforce'/, 'enforce must be the default mode'); + assert.match(src, /generateLegacySingleHeadlineBrief\(topStories\)/, 'L2 fallback must be wired'); + assert.match(src, /briefStoryLines/); + assert.match(src, /sourceAgeRange/); + }); + + it('country-intel brief strips invented citations before shipping', () => { + const src = readSrc('server/worldmonitor/intelligence/v1/get-country-intel-brief.ts'); + assert.match(src, /verifyCitationIndexes\(llmResult\.content, entrySources\.length\)/); + assert.match(src, /brief: citationCheck\.text/); + }); + + it('panel renders story lines and the freshness footer', () => { + const src = readSrc('src/components/InsightsPanel.ts'); + assert.match(src, /renderBriefExtras/); + assert.match(src, /insights-brief-lines/); + assert.match(src, /components\.insights\.briefFreshness/); + }); + + it('core and mirrors are byte-identical (grounding spine included)', () => { + assert.equal(readSrc('shared/brief-llm-core.js'), readSrc('scripts/shared/brief-llm-core.js')); + assert.equal(readSrc('shared/brief-llm-core.d.ts'), readSrc('scripts/shared/brief-llm-core.d.ts')); + }); +}); + +// ── #4928 review-round additions ─────────────────────────────────────────── + +import { composeSynthesizedBrief } from '../scripts/_insights-brief.mjs'; + +describe('composeSynthesizedBrief (functional L1 coverage, #4928 review)', () => { + const CORROBORATED = [ + { primaryTitle: 'Iran threatens to close Strait of Hormuz', primarySource: 'Reuters', primaryLink: 'https://r/1', pubDate: '2026-07-06T01:00:00Z', sources: ['Reuters', 'BBC'] }, + { primaryTitle: 'Turkey hikes interest rates to 50%', primarySource: 'Bloomberg', primaryLink: 'https://b/2', pubDate: '2026-07-06T02:00:00Z', sources: ['Bloomberg'] }, + ]; + const GOOD = JSON.stringify({ + lead: 'Iran raises the stakes around Hormuz [1] while Turkey delivers a dramatic rate hike [2].', + lines: [ + { n: 1, text: 'Iran threatens to close the Strait of Hormuz [1].' }, + { n: 2, text: 'Turkey raises interest rates to 50% [2].' }, + ], + }); + const passOpts = { validatorMode: 'enforce', sourceFromStory: (s) => ({ title: s.primaryTitle, source: s.primarySource, url: s.primaryLink }) }; + + it('happy path: lead + locked lines + lockstep sources', () => { + const out = composeSynthesizedBrief(GOOD, CORROBORATED, passOpts); + assert.ok(out); + assert.match(out.lead, /Hormuz \[1\]/); + assert.equal(out.lines.length, 2); + assert.equal(out.sources.length, 2); + assert.equal(out.sources[1].url, 'https://b/2'); + }); + + it('REGRESSION: a story without a usable link gets a substitute source entry, never shifting [n] mapping', () => { + const out = composeSynthesizedBrief(GOOD, CORROBORATED, { + ...passOpts, + sourceFromStory: (s) => (s.primarySource === 'Reuters' ? null : { title: s.primaryTitle, source: s.primarySource, url: s.primaryLink }), + }); + assert.ok(out); + assert.equal(out.sources.length, 2, 'sources must stay index-locked'); + assert.equal(out.sources[0].url, '', 'missing link → substitute entry, not filtered'); + assert.equal(out.sources[1].url, 'https://b/2', '[2] still points at story 2'); + }); + + it('editorial gate: all-single-source days reject L1 (legacy corroboration bar preserved)', () => { + const singles = CORROBORATED.map((s) => ({ ...s, sources: [s.primarySource] })); + assert.equal(composeSynthesizedBrief(GOOD, singles, passOpts), null); + }); + + it('lead inventing a proper noun is rejected in enforce mode (falls back)', () => { + const fabricated = JSON.stringify({ + lead: 'President Macron condemned the Hormuz escalation [1] as Turkey hiked rates [2].', + lines: [ + { n: 1, text: 'Iran threatens to close the Strait of Hormuz [1].' }, + { n: 2, text: 'Turkey raises interest rates to 50% [2].' }, + ], + }); + assert.equal(composeSynthesizedBrief(fabricated, CORROBORATED, passOpts), null); + }); + + it('a line inventing a proper noun degrades to its headline WITH its citation', () => { + const badLine = JSON.stringify({ + lead: 'Iran raises the stakes around Hormuz [1] while Turkey delivers a dramatic rate hike [2].', + lines: [ + { n: 1, text: 'Ayatollah Nasrallah vows to close the Strait of Hormuz [1].' }, + { n: 2, text: 'Turkey raises interest rates to 50% [2].' }, + ], + }); + const out = composeSynthesizedBrief(badLine, CORROBORATED, passOpts); + assert.ok(out); + assert.equal(out.hallucinatedLines, 1); + assert.equal(out.lines[0].text, 'Iran threatens to close Strait of Hormuz [1]', 'degraded line keeps [n]'); + }); + + it('missing line fills from headline with its citation', () => { + const partial = JSON.stringify({ + lead: 'Iran raises the stakes around Hormuz [1] while Turkey delivers a dramatic rate hike [2].', + lines: [{ n: 1, text: 'Iran threatens to close the Strait of Hormuz [1].' }], + }); + const out = composeSynthesizedBrief(partial, CORROBORATED, passOpts); + assert.ok(out); + assert.match(out.lines[1].text, /\[2\]$/); + }); +}); + +describe('boundary + contract pins (#4928 review)', () => { + it('parser lead-length bounds are inclusive at 40 and 700', () => { + const mk = (leadLen) => JSON.stringify({ + lead: 'L'.repeat(leadLen), + lines: [{ n: 1, text: 'A perfectly reasonable line for story one [1].' }], + }); + assert.ok(parseBriefSynthesis(mk(40), 1), '40-char lead must pass'); + assert.ok(parseBriefSynthesis(mk(700), 1), '700-char lead must pass'); + assert.equal(parseBriefSynthesis(mk(39), 1), null); + assert.equal(parseBriefSynthesis(mk(701), 1), null); + }); + + it('system prompt pins the exact JSON keys the parser reads', () => { + const prompt = synthesisSystemPrompt('2026-07-06'); + for (const key of ['"lead"', '"lines"', '"n"', '"text"']) { + assert.ok(prompt.includes(key), `prompt must name ${key} — parser depends on it`); + } + }); + + it('verifyCitationIndexes catches 3-digit invented markers, leaves 4-digit prose alone', () => { + const { text, stripped } = verifyCitationIndexes('Claim [123] and year [2026] and real [1].', 2); + assert.equal(stripped, 1, '[123] stripped'); + assert.match(text, /\[2026\]/, 'bracketed years are prose, not citations'); + assert.match(text, /\[1\]/); + }); +}); + +// ── #4928 external-review round ──────────────────────────────────────────── + +describe('citation-scoped composer gates (#4928 external review)', () => { + const STORIES2 = [ + { primaryTitle: 'Iran threatens to close Strait of Hormuz', primarySource: 'Reuters', primaryLink: 'https://r/1', pubDate: '2026-07-06T01:00:00Z', sources: ['Reuters', 'BBC'] }, + { primaryTitle: 'Turkey hikes interest rates to 50%', primarySource: 'Bloomberg', primaryLink: 'https://b/2', pubDate: '2026-07-06T02:00:00Z', sources: ['Bloomberg'] }, + ]; + const passOpts = { validatorMode: 'enforce', sourceFromStory: (s) => ({ title: s.primaryTitle, source: s.primarySource, url: s.primaryLink }) }; + + it('REGRESSION: a lead sentence attributing story-2 facts to [1] is rejected (misattribution)', () => { + const misattributed = JSON.stringify({ + lead: 'Turkey hikes interest rates to 50% in a dramatic move [1]. Iran threatens the Strait of Hormuz [1].', + lines: [ + { n: 1, text: 'Iran threatens to close the Strait of Hormuz [1].' }, + { n: 2, text: 'Turkey raises interest rates to 50% [2].' }, + ], + }); + assert.equal(composeSynthesizedBrief(misattributed, STORIES2, passOpts), null, + 'Turkey facts cited to [1] (Iran) must fail citation-scoped validation'); + }); + + it('REGRESSION: an uncited lead sentence rejects the synthesis (every claim cited)', () => { + const uncited = JSON.stringify({ + lead: 'Iran threatens the Strait of Hormuz [1]. Markets everywhere are nervous about what comes next.', + lines: [ + { n: 1, text: 'Iran threatens to close the Strait of Hormuz [1].' }, + { n: 2, text: 'Turkey raises interest rates to 50% [2].' }, + ], + }); + assert.equal(composeSynthesizedBrief(uncited, STORIES2, passOpts), null); + }); + + it('REGRESSION: a line carrying the WRONG in-range citation is rewritten to its own [n]', () => { + const wrongCite = JSON.stringify({ + lead: 'Iran raises the stakes around Hormuz [1] while Turkey delivers a dramatic rate hike [2].', + lines: [ + { n: 1, text: 'Iran threatens to close the Strait of Hormuz [1].' }, + { n: 2, text: 'Turkey raises interest rates to 50% [1].' }, + ], + }); + const out = composeSynthesizedBrief(wrongCite, STORIES2, passOpts); + assert.ok(out); + assert.match(out.lines[1].text, /\[2\]$/, 'line 2 must end with [2], never [1]'); + assert.doesNotMatch(out.lines[1].text.replace(/\[2\]$/, ''), /\[\d+\]/, 'foreign citations stripped'); + }); + + it('REGRESSION: a line with no surviving citation still ends with its own [n]', () => { + const uncitedLine = JSON.stringify({ + lead: 'Iran raises the stakes around Hormuz [1] while Turkey delivers a dramatic rate hike [2].', + lines: [ + { n: 1, text: 'Iran threatens to close the Strait of Hormuz [9].' }, + { n: 2, text: 'Turkey raises interest rates to 50% [2].' }, + ], + }); + const out = composeSynthesizedBrief(uncitedLine, STORIES2, passOpts); + assert.ok(out); + assert.match(out.lines[0].text, /\[1\]$/); + }); +}); + +describe('balanced-brace extraction (#4928 external review P3)', () => { + it('a stray closing brace in trailing prose no longer defeats the parse', () => { + const withStray = JSON.stringify({ + lead: 'Iran escalates around Hormuz [1] and markets brace for the fallout of it all [1].', + lines: [{ n: 1, text: 'Iran threatens to close the Strait of Hormuz [1].' }], + }) + '\nHope that helps! (edge case: })'; + assert.ok(parseBriefSynthesis(withStray, 1), 'stray } after the object must not break extraction'); + }); + + it('braces inside JSON strings do not confuse the scanner', () => { + const withInnerBrace = JSON.stringify({ + lead: 'Iran { escalates } around Hormuz [1] and markets brace for the fallout today [1].', + lines: [{ n: 1, text: 'Iran threatens to close the Strait of Hormuz [1].' }], + }); + assert.ok(parseBriefSynthesis(withInnerBrace, 1)); + }); +}); diff --git a/tests/insights-brief-sources-static.test.mts b/tests/insights-brief-sources-static.test.mts index eadb8adc52..2da1f78b46 100644 --- a/tests/insights-brief-sources-static.test.mts +++ b/tests/insights-brief-sources-static.test.mts @@ -6,9 +6,13 @@ const source = readFileSync(new URL('../src/components/InsightsPanel.ts', import describe('InsightsPanel server brief sources', () => { it('does not fabricate legacy world brief citations from topStories[0]', () => { + // #4928: the cap is now the payload's own citation index space + // (bounded 6..12) instead of a flat 6 — the invariant this test + // guards is unchanged: sources come ONLY from explicit + // worldBriefSources, never fabricated from topStories. assert.match( source, - /collectBriefSources\(\s*insights\.worldBriefSources \?\? \[\],\s*6,\s*\)/, + /collectBriefSources\(\s*insights\.worldBriefSources \?\? \[\],\s*Math\.min\(12, Math\.max\(6, insights\.worldBriefSources\?\.length \?\? 6\)\),\s*\)/, 'server-rendered world briefs should cite only explicit worldBriefSources', ); assert.doesNotMatch( diff --git a/tests/redis-caching.test.mjs b/tests/redis-caching.test.mjs index fbd7af9978..c112ea5014 100644 --- a/tests/redis-caching.test.mjs +++ b/tests/redis-caching.test.mjs @@ -804,6 +804,8 @@ describe('country risk freshness behavior', { concurrency: 1 }, () => { './_shared': resolve(root, 'server/worldmonitor/intelligence/v1/_shared.ts'), '../../../_shared/redis': resolve(root, 'server/_shared/redis.ts'), '../../../_shared/cache-keys': resolve(root, 'server/_shared/cache-keys.ts'), + // #4921: citation verification + grounding telemetry import + '../../../../shared/brief-llm-core.js': resolve(root, 'shared/brief-llm-core.js'), }); } @@ -1096,6 +1098,8 @@ describe('country intel brief caching behavior', { concurrency: 1 }, () => { ), '../../../_shared/llm-sanitize.js': resolve(root, 'server/_shared/llm-sanitize.js'), '../../../_shared/cache-keys': resolve(root, 'server/_shared/cache-keys.ts'), + // #4921: citation verification + grounding telemetry import + '../../../../shared/brief-llm-core.js': resolve(root, 'shared/brief-llm-core.js'), }); }