diff --git a/docs/api/NewsService.openapi.json b/docs/api/NewsService.openapi.json index c1bf5546c8..12d12493a1 100644 --- a/docs/api/NewsService.openapi.json +++ b/docs/api/NewsService.openapi.json @@ -1 +1 @@ -{"components":{"schemas":{"CategoriesEntry":{"properties":{"key":{"type":"string"},"value":{"$ref":"#/components/schemas/CategoryBucket"}},"type":"object"},"CategoryBucket":{"properties":{"items":{"items":{"$ref":"#/components/schemas/NewsItem"},"type":"array"}},"type":"object"},"Error":{"description":"Error is returned when a handler encounters an error. It contains a simple error message that the developer can customize.","properties":{"message":{"description":"Error message (e.g., 'user not found', 'database connection failed')","type":"string"}},"type":"object"},"FeedStatusesEntry":{"properties":{"key":{"type":"string"},"value":{"type":"string"}},"type":"object"},"FieldViolation":{"description":"FieldViolation describes a single validation error for a specific field.","properties":{"description":{"description":"Human-readable description of the validation violation (e.g., 'must be a valid email address', 'required field missing')","type":"string"},"field":{"description":"The field path that failed validation (e.g., 'user.email' for nested fields). For header validation, this will be the header name (e.g., 'X-API-Key')","type":"string"}},"required":["field","description"],"type":"object"},"ForbiddenError":{"description":"Returned when a PRO-gated endpoint denies access because the caller has no resolved authenticated user, entitlements cannot be verified, or the caller lacks the required entitlement tier.","properties":{"currentTier":{"description":"Caller entitlement tier when known.","format":"int32","type":"integer"},"error":{"description":"Human-readable entitlement failure reason.","type":"string"},"planKey":{"description":"Caller plan key when known.","type":"string"},"requiredTier":{"description":"Minimum entitlement tier required for this endpoint.","format":"int32","type":"integer"}},"required":["error"],"type":"object"},"GatewayError":{"description":"Returned by gateway infrastructure errors before an RPC handler runs, such as origin, routing, method, authentication, or quota checks.","properties":{"error":{"description":"Gateway error reason or structured gateway failure details.","oneOf":[{"type":"string"},{"additionalProperties":true,"type":"object"}]}},"required":["error"],"type":"object"},"GeoCoordinates":{"description":"GeoCoordinates represents a geographic location using WGS84 coordinates.","properties":{"latitude":{"description":"Latitude in decimal degrees (-90 to 90).","format":"double","maximum":90,"minimum":-90,"type":"number"},"longitude":{"description":"Longitude in decimal degrees (-180 to 180).","format":"double","maximum":180,"minimum":-180,"type":"number"}},"type":"object"},"GetSummarizeArticleCacheRequest":{"description":"GetSummarizeArticleCacheRequest looks up a pre-computed summary by cache key.","properties":{"cacheKey":{"description":"Deterministic cache key computed by buildSummaryCacheKey().","type":"string"}},"type":"object"},"InvalidRequestBodyError":{"description":"Returned when a JSON POST request body is empty or malformed.","properties":{"message":{"description":"Invalid request body","type":"string"}},"required":["message"],"type":"object"},"JmespathProjectionError":{"description":"Returned when a REST jmespath projection is invalid or exceeds the expression/output byte limits.","properties":{"_jmespath_error":{"description":"Projection error discriminator and details.","type":"string"},"original_keys":{"description":"Top-level keys or shape of the unprojected response.","items":{"type":"string"},"type":"array"}},"required":["_jmespath_error","original_keys"],"type":"object"},"ListFeedDigestRequest":{"properties":{"lang":{"description":"ISO 639-1 language code (en, fr, ar, etc.)","type":"string"},"variant":{"description":"Digest variant: full, tech, finance, happy, commodity. Unsupported site variants, including energy, currently fall back to full.","type":"string"}},"type":"object"},"ListFeedDigestResponse":{"properties":{"categories":{"additionalProperties":{"$ref":"#/components/schemas/CategoryBucket"},"description":"Per-category buckets — keys match category names from feed config","type":"object"},"feedStatuses":{"additionalProperties":{"type":"string"},"description":"Per-feed status — only non-ok states emitted; absent key implies ok.\n Values: empty (feed returned 0 items), timeout (timed out during fetch),\n all-undated (every parsed item lacked a usable date), partial-undated\n (some parsed items lacked a usable date).","type":"object"},"generatedAt":{"description":"ISO 8601 timestamp of when this digest was generated","type":"string"}},"type":"object"},"NewsItem":{"description":"NewsItem represents a single news article from RSS feed aggregation.","properties":{"corroborationCount":{"description":"Number of distinct sources that reported the same story in this digest cycle.","format":"int32","type":"integer"},"importanceScore":{"description":"Composite importance score. The base 0-100 score uses severity × 55% +\n source tier × 20% + corroboration × 15% + recency × 10%, then may add an\n 18-point diplomacy/flashpoint boost and 4 points per entity-level\n corroborating source, capped at five sources. The final score can exceed\n 100; with current boosts it is approximately capped at 138.","format":"int32","type":"integer"},"isAlert":{"description":"Whether this article triggered an alert condition.","type":"boolean"},"link":{"description":"Article URL.","type":"string"},"location":{"$ref":"#/components/schemas/GeoCoordinates"},"locationName":{"description":"Human-readable location name.","type":"string"},"publishedAt":{"description":"Publication time, as Unix epoch milliseconds.. Warning: Values \u003e 2^53 may lose precision in JavaScript","format":"int64","type":"integer"},"snippet":{"description":"Cleaned article description from the RSS/Atom \u003cdescription\u003e /\n \u003ccontent:encoded\u003e / \u003csummary\u003e / \u003ccontent\u003e tag: HTML-stripped,\n entity-decoded, whitespace-normalised, clipped to 400 chars. Empty string\n when unavailable or indistinguishable from the headline — consumers must\n fall back to the headline for display/LLM grounding in that case.","type":"string"},"source":{"description":"Source feed name.","minLength":1,"type":"string"},"storyMeta":{"$ref":"#/components/schemas/StoryMeta"},"threat":{"$ref":"#/components/schemas/ThreatClassification"},"title":{"description":"Article headline.","minLength":1,"type":"string"}},"required":["source","title"],"type":"object"},"RateLimitError":{"description":"Returned when a gateway or handler rate limit rejects the request.","properties":{"error":{"description":"Human-readable rate-limit failure reason.","type":"string"}},"required":["error"],"type":"object"},"StoryMeta":{"description":"StoryMeta carries cross-cycle persistence data attached to each news item.","properties":{"firstSeen":{"description":"Epoch ms when the story first appeared in any digest cycle.. Warning: Values \u003e 2^53 may lose precision in JavaScript","format":"int64","type":"integer"},"mentionCount":{"description":"Total number of digest cycles in which this story appeared.","format":"int32","type":"integer"},"phase":{"description":"StoryPhase represents the lifecycle stage of a tracked news story.","enum":["STORY_PHASE_UNSPECIFIED","STORY_PHASE_BREAKING","STORY_PHASE_DEVELOPING","STORY_PHASE_SUSTAINED","STORY_PHASE_FADING"],"type":"string"},"sourceCount":{"description":"Number of unique sources that reported this story (cached from Redis Set).","format":"int32","type":"integer"}},"type":"object"},"SummarizeArticleRequest":{"description":"SummarizeArticleRequest specifies parameters for LLM article summarization.","properties":{"bodies":{"items":{"description":"Optional article bodies paired 1:1 with `headlines`. When bodies[i] is\n non-empty, the prompt interleaves it as grounding context under\n headlines[i]; when empty, behavior is identical to headline-only today.\n Callers may supply a shorter array; missing entries are treated as empty.\n Each body is subject to the same sanitisation as headlines before reaching\n the LLM prompt.","type":"string"},"type":"array"},"geoContext":{"description":"Geographic signal context to include in the prompt.","type":"string"},"headlines":{"items":{"description":"Headlines to summarize. Up to 10 raw headlines are used for request\n identity/cache matching; the LLM prompt uses up to 5 unique, non-empty\n sanitized headline/body pairs.","minItems":1,"type":"string"},"minItems":1,"type":"array"},"lang":{"description":"Output language code, default \"en\".","type":"string"},"mode":{"description":"Summarization mode: \"brief\", \"analysis\", \"translate\", \"\" (default).","type":"string"},"provider":{"description":"LLM provider: \"ollama\", \"groq\", \"openrouter\"","minLength":1,"type":"string"},"systemAppend":{"description":"Optional system prompt append for analytical framework instructions.","type":"string"},"variant":{"description":"Variant: \"full\", \"tech\", or target language for translate mode.","type":"string"}},"required":["provider"],"type":"object"},"SummarizeArticleResponse":{"description":"SummarizeArticleResponse contains the LLM summarization result.","properties":{"error":{"description":"Error message if the request failed.","type":"string"},"errorType":{"description":"Error type/name (e.g. \"TypeError\").","type":"string"},"fallback":{"description":"Whether the client should try the next provider in the fallback chain.","type":"boolean"},"model":{"description":"Model identifier used for generation.","type":"string"},"provider":{"description":"Provider that produced the result (or \"cache\").","type":"string"},"status":{"description":"SummarizeStatus indicates the outcome of a summarization request.","enum":["SUMMARIZE_STATUS_UNSPECIFIED","SUMMARIZE_STATUS_SUCCESS","SUMMARIZE_STATUS_CACHED","SUMMARIZE_STATUS_SKIPPED","SUMMARIZE_STATUS_ERROR"],"type":"string"},"statusDetail":{"description":"Human-readable detail for non-success statuses (skip reason, etc.).","type":"string"},"summary":{"description":"The generated summary text.","type":"string"},"tokens":{"description":"Token count from the LLM response.","format":"int32","type":"integer"}},"type":"object"},"ThreatClassification":{"description":"ThreatClassification represents an AI-assessed threat level for a news item.","properties":{"category":{"description":"Event category.","type":"string"},"confidence":{"description":"Confidence score (0.0 to 1.0).","format":"double","maximum":1,"minimum":0,"type":"number"},"level":{"description":"ThreatLevel represents the assessed threat level of a news event.","enum":["THREAT_LEVEL_UNSPECIFIED","THREAT_LEVEL_LOW","THREAT_LEVEL_MEDIUM","THREAT_LEVEL_HIGH","THREAT_LEVEL_CRITICAL"],"type":"string"},"source":{"description":"Classification source — \"keyword\", \"keyword-historical-downgrade\", or\n \"llm\".","type":"string"}},"type":"object"},"UnauthorizedError":{"description":"Returned when the API key is missing, malformed, or lacks current API access.","properties":{"error":{"description":"Human-readable error message.","type":"string"}},"required":["error"],"type":"object"},"ValidationError":{"description":"ValidationError is returned when request validation fails. It contains a list of field violations describing what went wrong.","properties":{"violations":{"description":"List of validation violations","items":{"$ref":"#/components/schemas/FieldViolation"},"type":"array"}},"required":["violations"],"type":"object"}},"securitySchemes":{"ApiKeyHeader":{"description":"Alias header for the WorldMonitor API key (X-WorldMonitor-Key).","in":"header","name":"X-Api-Key","type":"apiKey"},"WorldMonitorKey":{"description":"User-issued WorldMonitor API key.","in":"header","name":"X-WorldMonitor-Key","type":"apiKey"}}},"info":{"title":"NewsService API","version":"1.0.0"},"openapi":"3.1.0","paths":{"/api/news/v1/list-feed-digest":{"get":{"description":"ListFeedDigest returns a pre-aggregated digest of all RSS feeds for a site variant.","operationId":"ListFeedDigest","parameters":[{"description":"Digest variant: full, tech, finance, happy, commodity. Unsupported site variants, including energy, currently fall back to full.","example":"example","in":"query","name":"variant","required":false,"schema":{"type":"string"}},{"description":"ISO 639-1 language code (en, fr, ar, etc.)","example":"en","in":"query","name":"lang","required":false,"schema":{"type":"string"}},{"description":"Optional JMESPath expression applied server-side to project or reduce the JSON response before it is returned (mirrors the MCP jmespath argument). Invalid expressions, expressions larger than 1024 UTF-8 bytes, or projections that exceed the 256 KB output cap return HTTP 400 with a {_jmespath_error, original_keys} envelope. Grammar and worked examples: https://www.worldmonitor.app/docs/mcp-jmespath.","example":"keys(@)","in":"query","name":"jmespath","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"application/json":{"example":{"categories":{"exampleKey":{"items":[{"corroborationCount":42,"importanceScore":42,"isAlert":true,"link":"https://example.com/worldmonitor","location":{"latitude":40.7128,"longitude":-74.006},"source":"example","title":"example"}]}},"feedStatuses":{"exampleKey":"example"},"generatedAt":"2026-01-15T12:00:00Z"},"schema":{"$ref":"#/components/schemas/ListFeedDigestResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"oneOf":[{"$ref":"#/components/schemas/ValidationError"},{"$ref":"#/components/schemas/JmespathProjectionError"}]}}},"description":"Validation error"},"401":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UnauthorizedError"}}},"description":"Missing or invalid API key."},"403":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ForbiddenError"}}},"description":"API access requires an active subscription (the API key's subscription is inactive or expired)."},"429":{"content":{"application/json":{"schema":{"oneOf":[{"$ref":"#/components/schemas/Error"},{"$ref":"#/components/schemas/RateLimitError"}]}}},"description":"Rate limit exceeded.","headers":{"Retry-After":{"description":"Seconds to wait before retrying the request.","schema":{"type":"string"}},"X-RateLimit-Limit":{"description":"Maximum requests allowed in the active rate-limit window.","schema":{"type":"string"}},"X-RateLimit-Remaining":{"description":"Requests remaining in the active rate-limit window.","schema":{"type":"string"}},"X-RateLimit-Reset":{"description":"Unix epoch milliseconds when the active rate-limit window resets.","schema":{"type":"string"}}}},"default":{"content":{"application/json":{"schema":{"oneOf":[{"$ref":"#/components/schemas/Error"},{"$ref":"#/components/schemas/GatewayError"}]}}},"description":"Gateway or handler error response."}},"summary":"ListFeedDigest","tags":["NewsService"]}},"/api/news/v1/summarize-article":{"post":{"description":"SummarizeArticle generates an LLM summary with provider selection and fallback support.","operationId":"SummarizeArticle","parameters":[{"description":"Optional client-generated idempotency key. Retrying a POST with the same key and an identical request body replays the original response (only the status, body, and Content-Type are reproduced) instead of re-executing; reusing the key with a different body is rejected with 422. For mutations this avoids duplicating the side effect, while for batch-read POSTs it replays a cached snapshot that can be up to 24 hours stale. Keys are scoped per authenticated caller (falling back to the source IP for unauthenticated endpoints) and retained for 24 hours.","example":"4f8b9c2e-1a3d-4b6f-8e0a-2c5d7f9b1e34","in":"header","name":"Idempotency-Key","required":false,"schema":{"maxLength":255,"minLength":1,"pattern":"^[\\x21-\\x7E]{1,255}$","type":"string"}}],"requestBody":{"content":{"application/json":{"example":{"bodies":["example"],"geoContext":"example","headlines":["example"],"lang":"en","mode":"brief","provider":"openrouter"},"schema":{"$ref":"#/components/schemas/SummarizeArticleRequest"}}},"required":true},"responses":{"200":{"content":{"application/json":{"example":{"error":"example","errorType":"all","fallback":true,"model":"example","provider":"openrouter"},"schema":{"$ref":"#/components/schemas/SummarizeArticleResponse"}}},"description":"Successful response","headers":{"Idempotency-Key":{"description":"The idempotency key echoed from the request. Present only when the client opted into idempotency.","schema":{"type":"string"}},"Idempotent-Replayed":{"description":"true when this response was replayed from an earlier request with the same key, false on the first (original) request. Present only when the client opted into idempotency.","schema":{"type":"boolean"}}}},"400":{"content":{"application/json":{"schema":{"oneOf":[{"$ref":"#/components/schemas/ValidationError"},{"properties":{"error":{"type":"string"},"message":{"type":"string"}},"required":["error","message"],"type":"object"},{"$ref":"#/components/schemas/InvalidRequestBodyError"}]}}},"description":"Validation error, invalid Idempotency-Key header, or malformed JSON request body"},"401":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UnauthorizedError"}}},"description":"Missing or invalid API key."},"403":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ForbiddenError"}}},"description":"API access requires an active subscription (the API key's subscription is inactive or expired)."},"409":{"content":{"application/json":{"schema":{"properties":{"error":{"type":"string"},"message":{"type":"string"}},"required":["error","message"],"type":"object"}}},"description":"A request with this Idempotency-Key is still being processed","headers":{"Idempotency-Key":{"description":"The idempotency key supplied by the client.","schema":{"type":"string"}},"Retry-After":{"description":"Seconds to wait before retrying the in-flight request.","schema":{"type":"string"}}}},"422":{"content":{"application/json":{"schema":{"properties":{"error":{"type":"string"},"message":{"type":"string"}},"required":["error","message"],"type":"object"}}},"description":"The Idempotency-Key was already used with a different request body","headers":{"Idempotency-Key":{"description":"The idempotency key supplied by the client.","schema":{"type":"string"}}}},"429":{"content":{"application/json":{"schema":{"oneOf":[{"$ref":"#/components/schemas/Error"},{"$ref":"#/components/schemas/RateLimitError"}]}}},"description":"Rate limit exceeded.","headers":{"Retry-After":{"description":"Seconds to wait before retrying the request.","schema":{"type":"string"}},"X-RateLimit-Limit":{"description":"Maximum requests allowed in the active rate-limit window.","schema":{"type":"string"}},"X-RateLimit-Remaining":{"description":"Requests remaining in the active rate-limit window.","schema":{"type":"string"}},"X-RateLimit-Reset":{"description":"Unix epoch milliseconds when the active rate-limit window resets.","schema":{"type":"string"}}}},"default":{"content":{"application/json":{"schema":{"oneOf":[{"$ref":"#/components/schemas/Error"},{"$ref":"#/components/schemas/GatewayError"}]}}},"description":"Gateway or handler error response."}},"summary":"SummarizeArticle","tags":["NewsService"]}},"/api/news/v1/summarize-article-cache":{"get":{"description":"GetSummarizeArticleCache looks up a cached summary by deterministic key (CDN-cacheable GET).","operationId":"GetSummarizeArticleCache","parameters":[{"description":"Deterministic cache key computed by buildSummaryCacheKey().","example":"summary:v1:example-cache","in":"query","name":"cache_key","required":false,"schema":{"pattern":"^summary:v\\d+:[a-z0-9:_-]{3,120}$","type":"string"}},{"description":"Optional JMESPath expression applied server-side to project or reduce the JSON response before it is returned (mirrors the MCP jmespath argument). Invalid expressions, expressions larger than 1024 UTF-8 bytes, or projections that exceed the 256 KB output cap return HTTP 400 with a {_jmespath_error, original_keys} envelope. Grammar and worked examples: https://www.worldmonitor.app/docs/mcp-jmespath.","example":"keys(@)","in":"query","name":"jmespath","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"application/json":{"example":{"error":"example","errorType":"all","fallback":true,"model":"example","provider":"openrouter"},"schema":{"$ref":"#/components/schemas/SummarizeArticleResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"oneOf":[{"$ref":"#/components/schemas/ValidationError"},{"$ref":"#/components/schemas/JmespathProjectionError"}]}}},"description":"Validation error"},"401":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UnauthorizedError"}}},"description":"Missing or invalid API key."},"403":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ForbiddenError"}}},"description":"API access requires an active subscription (the API key's subscription is inactive or expired)."},"429":{"content":{"application/json":{"schema":{"oneOf":[{"$ref":"#/components/schemas/Error"},{"$ref":"#/components/schemas/RateLimitError"}]}}},"description":"Rate limit exceeded.","headers":{"Retry-After":{"description":"Seconds to wait before retrying the request.","schema":{"type":"string"}},"X-RateLimit-Limit":{"description":"Maximum requests allowed in the active rate-limit window.","schema":{"type":"string"}},"X-RateLimit-Remaining":{"description":"Requests remaining in the active rate-limit window.","schema":{"type":"string"}},"X-RateLimit-Reset":{"description":"Unix epoch milliseconds when the active rate-limit window resets.","schema":{"type":"string"}}}},"default":{"content":{"application/json":{"schema":{"oneOf":[{"$ref":"#/components/schemas/Error"},{"$ref":"#/components/schemas/GatewayError"}]}}},"description":"Gateway or handler error response."}},"summary":"GetSummarizeArticleCache","tags":["NewsService"]}}},"security":[{"WorldMonitorKey":[]},{"ApiKeyHeader":[]}],"servers":[{"url":"https://api.worldmonitor.app"}]} \ No newline at end of file +{"components":{"schemas":{"CategoriesEntry":{"properties":{"key":{"type":"string"},"value":{"$ref":"#/components/schemas/CategoryBucket"}},"type":"object"},"CategoryBucket":{"properties":{"items":{"items":{"$ref":"#/components/schemas/NewsItem"},"type":"array"}},"type":"object"},"Error":{"description":"Error is returned when a handler encounters an error. It contains a simple error message that the developer can customize.","properties":{"message":{"description":"Error message (e.g., 'user not found', 'database connection failed')","type":"string"}},"type":"object"},"FeedStatusesEntry":{"properties":{"key":{"type":"string"},"value":{"type":"string"}},"type":"object"},"FieldViolation":{"description":"FieldViolation describes a single validation error for a specific field.","properties":{"description":{"description":"Human-readable description of the validation violation (e.g., 'must be a valid email address', 'required field missing')","type":"string"},"field":{"description":"The field path that failed validation (e.g., 'user.email' for nested fields). For header validation, this will be the header name (e.g., 'X-API-Key')","type":"string"}},"required":["field","description"],"type":"object"},"ForbiddenError":{"description":"Returned when a PRO-gated endpoint denies access because the caller has no resolved authenticated user, entitlements cannot be verified, or the caller lacks the required entitlement tier.","properties":{"currentTier":{"description":"Caller entitlement tier when known.","format":"int32","type":"integer"},"error":{"description":"Human-readable entitlement failure reason.","type":"string"},"planKey":{"description":"Caller plan key when known.","type":"string"},"requiredTier":{"description":"Minimum entitlement tier required for this endpoint.","format":"int32","type":"integer"}},"required":["error"],"type":"object"},"GatewayError":{"description":"Returned by gateway infrastructure errors before an RPC handler runs, such as origin, routing, method, authentication, or quota checks.","properties":{"error":{"description":"Gateway error reason or structured gateway failure details.","oneOf":[{"type":"string"},{"additionalProperties":true,"type":"object"}]}},"required":["error"],"type":"object"},"GeoCoordinates":{"description":"GeoCoordinates represents a geographic location using WGS84 coordinates.","properties":{"latitude":{"description":"Latitude in decimal degrees (-90 to 90).","format":"double","maximum":90,"minimum":-90,"type":"number"},"longitude":{"description":"Longitude in decimal degrees (-180 to 180).","format":"double","maximum":180,"minimum":-180,"type":"number"}},"type":"object"},"GetSummarizeArticleCacheRequest":{"description":"GetSummarizeArticleCacheRequest looks up a pre-computed summary by cache key.","properties":{"cacheKey":{"description":"Deterministic cache key computed by buildSummaryCacheKey().","type":"string"}},"type":"object"},"InvalidRequestBodyError":{"description":"Returned when a JSON POST request body is empty or malformed.","properties":{"message":{"description":"Invalid request body","type":"string"}},"required":["message"],"type":"object"},"JmespathProjectionError":{"description":"Returned when a REST jmespath projection is invalid or exceeds the expression/output byte limits.","properties":{"_jmespath_error":{"description":"Projection error discriminator and details.","type":"string"},"original_keys":{"description":"Top-level keys or shape of the unprojected response.","items":{"type":"string"},"type":"array"}},"required":["_jmespath_error","original_keys"],"type":"object"},"ListFeedDigestRequest":{"properties":{"lang":{"description":"ISO 639-1 language code (en, fr, ar, etc.)","type":"string"},"variant":{"description":"Digest variant: full, tech, finance, happy, commodity. Unsupported site variants, including energy, currently fall back to full.","type":"string"}},"type":"object"},"ListFeedDigestResponse":{"properties":{"categories":{"additionalProperties":{"$ref":"#/components/schemas/CategoryBucket"},"description":"Per-category buckets — keys match category names from feed config","type":"object"},"feedStatuses":{"additionalProperties":{"type":"string"},"description":"Per-feed status — only non-ok states emitted; absent key implies ok.\n Values: empty (feed returned 0 items), timeout (timed out during fetch),\n all-undated (every parsed item lacked a usable date), partial-undated\n (some parsed items lacked a usable date).","type":"object"},"generatedAt":{"description":"ISO 8601 timestamp of when this digest was generated","type":"string"}},"type":"object"},"NewsItem":{"description":"NewsItem represents a single news article from RSS feed aggregation.","properties":{"corroborationCount":{"description":"Number of distinct sources that reported the same story in this digest\n cycle. \"Same story\" is determined by edit-tolerant story-identity\n clustering (headlines describing one event with different wording,\n ordering, source suffixes, truncation, or morphology corroborate each\n other) — not exact-title matching. See shared/story-identity.js.","format":"int32","type":"integer"},"importanceScore":{"description":"Composite importance score. The base 0-100 score uses severity × 55% +\n source tier × 20% + corroboration × 15% + recency × 10%, then may add an\n 18-point diplomacy/flashpoint boost and 4 points per entity-level\n corroborating source, capped at five sources. The final score can exceed\n 100; with current boosts it is approximately capped at 138.","format":"int32","type":"integer"},"isAlert":{"description":"Whether this article triggered an alert condition.","type":"boolean"},"link":{"description":"Article URL.","type":"string"},"location":{"$ref":"#/components/schemas/GeoCoordinates"},"locationName":{"description":"Human-readable location name.","type":"string"},"publishedAt":{"description":"Publication time, as Unix epoch milliseconds.. Warning: Values \u003e 2^53 may lose precision in JavaScript","format":"int64","type":"integer"},"snippet":{"description":"Cleaned article description from the RSS/Atom \u003cdescription\u003e /\n \u003ccontent:encoded\u003e / \u003csummary\u003e / \u003ccontent\u003e tag: HTML-stripped,\n entity-decoded, whitespace-normalised, clipped to 400 chars. Empty string\n when unavailable or indistinguishable from the headline — consumers must\n fall back to the headline for display/LLM grounding in that case.","type":"string"},"source":{"description":"Source feed name.","minLength":1,"type":"string"},"storyMeta":{"$ref":"#/components/schemas/StoryMeta"},"threat":{"$ref":"#/components/schemas/ThreatClassification"},"title":{"description":"Article headline.","minLength":1,"type":"string"}},"required":["source","title"],"type":"object"},"RateLimitError":{"description":"Returned when a gateway or handler rate limit rejects the request.","properties":{"error":{"description":"Human-readable rate-limit failure reason.","type":"string"}},"required":["error"],"type":"object"},"StoryMeta":{"description":"StoryMeta carries cross-cycle persistence data attached to each news item.","properties":{"firstSeen":{"description":"Epoch ms when the story first appeared in any digest cycle.. Warning: Values \u003e 2^53 may lose precision in JavaScript","format":"int64","type":"integer"},"mentionCount":{"description":"Total number of digest cycles in which this story appeared.","format":"int32","type":"integer"},"phase":{"description":"StoryPhase represents the lifecycle stage of a tracked news story.","enum":["STORY_PHASE_UNSPECIFIED","STORY_PHASE_BREAKING","STORY_PHASE_DEVELOPING","STORY_PHASE_SUSTAINED","STORY_PHASE_FADING"],"type":"string"},"sourceCount":{"description":"Number of unique sources that reported this story (cached from Redis Set).","format":"int32","type":"integer"}},"type":"object"},"SummarizeArticleRequest":{"description":"SummarizeArticleRequest specifies parameters for LLM article summarization.","properties":{"bodies":{"items":{"description":"Optional article bodies paired 1:1 with `headlines`. When bodies[i] is\n non-empty, the prompt interleaves it as grounding context under\n headlines[i]; when empty, behavior is identical to headline-only today.\n Callers may supply a shorter array; missing entries are treated as empty.\n Each body is subject to the same sanitisation as headlines before reaching\n the LLM prompt.","type":"string"},"type":"array"},"geoContext":{"description":"Geographic signal context to include in the prompt.","type":"string"},"headlines":{"items":{"description":"Headlines to summarize. Up to 10 raw headlines are used for request\n identity/cache matching; the LLM prompt uses up to 5 unique, non-empty\n sanitized headline/body pairs.","minItems":1,"type":"string"},"minItems":1,"type":"array"},"lang":{"description":"Output language code, default \"en\".","type":"string"},"mode":{"description":"Summarization mode: \"brief\", \"analysis\", \"translate\", \"\" (default).","type":"string"},"provider":{"description":"LLM provider: \"ollama\", \"groq\", \"openrouter\"","minLength":1,"type":"string"},"systemAppend":{"description":"Optional system prompt append for analytical framework instructions.","type":"string"},"variant":{"description":"Variant: \"full\", \"tech\", or target language for translate mode.","type":"string"}},"required":["provider"],"type":"object"},"SummarizeArticleResponse":{"description":"SummarizeArticleResponse contains the LLM summarization result.","properties":{"error":{"description":"Error message if the request failed.","type":"string"},"errorType":{"description":"Error type/name (e.g. \"TypeError\").","type":"string"},"fallback":{"description":"Whether the client should try the next provider in the fallback chain.","type":"boolean"},"model":{"description":"Model identifier used for generation.","type":"string"},"provider":{"description":"Provider that produced the result (or \"cache\").","type":"string"},"status":{"description":"SummarizeStatus indicates the outcome of a summarization request.","enum":["SUMMARIZE_STATUS_UNSPECIFIED","SUMMARIZE_STATUS_SUCCESS","SUMMARIZE_STATUS_CACHED","SUMMARIZE_STATUS_SKIPPED","SUMMARIZE_STATUS_ERROR"],"type":"string"},"statusDetail":{"description":"Human-readable detail for non-success statuses (skip reason, etc.).","type":"string"},"summary":{"description":"The generated summary text.","type":"string"},"tokens":{"description":"Token count from the LLM response.","format":"int32","type":"integer"}},"type":"object"},"ThreatClassification":{"description":"ThreatClassification represents an AI-assessed threat level for a news item.","properties":{"category":{"description":"Event category.","type":"string"},"confidence":{"description":"Confidence score (0.0 to 1.0).","format":"double","maximum":1,"minimum":0,"type":"number"},"level":{"description":"ThreatLevel represents the assessed threat level of a news event.","enum":["THREAT_LEVEL_UNSPECIFIED","THREAT_LEVEL_LOW","THREAT_LEVEL_MEDIUM","THREAT_LEVEL_HIGH","THREAT_LEVEL_CRITICAL"],"type":"string"},"source":{"description":"Classification source — \"keyword\", \"keyword-historical-downgrade\", or\n \"llm\".","type":"string"}},"type":"object"},"UnauthorizedError":{"description":"Returned when the API key is missing, malformed, or lacks current API access.","properties":{"error":{"description":"Human-readable error message.","type":"string"}},"required":["error"],"type":"object"},"ValidationError":{"description":"ValidationError is returned when request validation fails. It contains a list of field violations describing what went wrong.","properties":{"violations":{"description":"List of validation violations","items":{"$ref":"#/components/schemas/FieldViolation"},"type":"array"}},"required":["violations"],"type":"object"}},"securitySchemes":{"ApiKeyHeader":{"description":"Alias header for the WorldMonitor API key (X-WorldMonitor-Key).","in":"header","name":"X-Api-Key","type":"apiKey"},"WorldMonitorKey":{"description":"User-issued WorldMonitor API key.","in":"header","name":"X-WorldMonitor-Key","type":"apiKey"}}},"info":{"title":"NewsService API","version":"1.0.0"},"openapi":"3.1.0","paths":{"/api/news/v1/list-feed-digest":{"get":{"description":"ListFeedDigest returns a pre-aggregated digest of all RSS feeds for a site variant.","operationId":"ListFeedDigest","parameters":[{"description":"Digest variant: full, tech, finance, happy, commodity. Unsupported site variants, including energy, currently fall back to full.","example":"example","in":"query","name":"variant","required":false,"schema":{"type":"string"}},{"description":"ISO 639-1 language code (en, fr, ar, etc.)","example":"en","in":"query","name":"lang","required":false,"schema":{"type":"string"}},{"description":"Optional JMESPath expression applied server-side to project or reduce the JSON response before it is returned (mirrors the MCP jmespath argument). Invalid expressions, expressions larger than 1024 UTF-8 bytes, or projections that exceed the 256 KB output cap return HTTP 400 with a {_jmespath_error, original_keys} envelope. Grammar and worked examples: https://www.worldmonitor.app/docs/mcp-jmespath.","example":"keys(@)","in":"query","name":"jmespath","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"application/json":{"example":{"categories":{"exampleKey":{"items":[{"corroborationCount":42,"importanceScore":42,"isAlert":true,"link":"https://example.com/worldmonitor","location":{"latitude":40.7128,"longitude":-74.006},"source":"example","title":"example"}]}},"feedStatuses":{"exampleKey":"example"},"generatedAt":"2026-01-15T12:00:00Z"},"schema":{"$ref":"#/components/schemas/ListFeedDigestResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"oneOf":[{"$ref":"#/components/schemas/ValidationError"},{"$ref":"#/components/schemas/JmespathProjectionError"}]}}},"description":"Validation error"},"401":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UnauthorizedError"}}},"description":"Missing or invalid API key."},"403":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ForbiddenError"}}},"description":"API access requires an active subscription (the API key's subscription is inactive or expired)."},"429":{"content":{"application/json":{"schema":{"oneOf":[{"$ref":"#/components/schemas/Error"},{"$ref":"#/components/schemas/RateLimitError"}]}}},"description":"Rate limit exceeded.","headers":{"Retry-After":{"description":"Seconds to wait before retrying the request.","schema":{"type":"string"}},"X-RateLimit-Limit":{"description":"Maximum requests allowed in the active rate-limit window.","schema":{"type":"string"}},"X-RateLimit-Remaining":{"description":"Requests remaining in the active rate-limit window.","schema":{"type":"string"}},"X-RateLimit-Reset":{"description":"Unix epoch milliseconds when the active rate-limit window resets.","schema":{"type":"string"}}}},"default":{"content":{"application/json":{"schema":{"oneOf":[{"$ref":"#/components/schemas/Error"},{"$ref":"#/components/schemas/GatewayError"}]}}},"description":"Gateway or handler error response."}},"summary":"ListFeedDigest","tags":["NewsService"]}},"/api/news/v1/summarize-article":{"post":{"description":"SummarizeArticle generates an LLM summary with provider selection and fallback support.","operationId":"SummarizeArticle","parameters":[{"description":"Optional client-generated idempotency key. Retrying a POST with the same key and an identical request body replays the original response (only the status, body, and Content-Type are reproduced) instead of re-executing; reusing the key with a different body is rejected with 422. For mutations this avoids duplicating the side effect, while for batch-read POSTs it replays a cached snapshot that can be up to 24 hours stale. Keys are scoped per authenticated caller (falling back to the source IP for unauthenticated endpoints) and retained for 24 hours.","example":"4f8b9c2e-1a3d-4b6f-8e0a-2c5d7f9b1e34","in":"header","name":"Idempotency-Key","required":false,"schema":{"maxLength":255,"minLength":1,"pattern":"^[\\x21-\\x7E]{1,255}$","type":"string"}}],"requestBody":{"content":{"application/json":{"example":{"bodies":["example"],"geoContext":"example","headlines":["example"],"lang":"en","mode":"brief","provider":"openrouter"},"schema":{"$ref":"#/components/schemas/SummarizeArticleRequest"}}},"required":true},"responses":{"200":{"content":{"application/json":{"example":{"error":"example","errorType":"all","fallback":true,"model":"example","provider":"openrouter"},"schema":{"$ref":"#/components/schemas/SummarizeArticleResponse"}}},"description":"Successful response","headers":{"Idempotency-Key":{"description":"The idempotency key echoed from the request. Present only when the client opted into idempotency.","schema":{"type":"string"}},"Idempotent-Replayed":{"description":"true when this response was replayed from an earlier request with the same key, false on the first (original) request. Present only when the client opted into idempotency.","schema":{"type":"boolean"}}}},"400":{"content":{"application/json":{"schema":{"oneOf":[{"$ref":"#/components/schemas/ValidationError"},{"properties":{"error":{"type":"string"},"message":{"type":"string"}},"required":["error","message"],"type":"object"},{"$ref":"#/components/schemas/InvalidRequestBodyError"}]}}},"description":"Validation error, invalid Idempotency-Key header, or malformed JSON request body"},"401":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UnauthorizedError"}}},"description":"Missing or invalid API key."},"403":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ForbiddenError"}}},"description":"API access requires an active subscription (the API key's subscription is inactive or expired)."},"409":{"content":{"application/json":{"schema":{"properties":{"error":{"type":"string"},"message":{"type":"string"}},"required":["error","message"],"type":"object"}}},"description":"A request with this Idempotency-Key is still being processed","headers":{"Idempotency-Key":{"description":"The idempotency key supplied by the client.","schema":{"type":"string"}},"Retry-After":{"description":"Seconds to wait before retrying the in-flight request.","schema":{"type":"string"}}}},"422":{"content":{"application/json":{"schema":{"properties":{"error":{"type":"string"},"message":{"type":"string"}},"required":["error","message"],"type":"object"}}},"description":"The Idempotency-Key was already used with a different request body","headers":{"Idempotency-Key":{"description":"The idempotency key supplied by the client.","schema":{"type":"string"}}}},"429":{"content":{"application/json":{"schema":{"oneOf":[{"$ref":"#/components/schemas/Error"},{"$ref":"#/components/schemas/RateLimitError"}]}}},"description":"Rate limit exceeded.","headers":{"Retry-After":{"description":"Seconds to wait before retrying the request.","schema":{"type":"string"}},"X-RateLimit-Limit":{"description":"Maximum requests allowed in the active rate-limit window.","schema":{"type":"string"}},"X-RateLimit-Remaining":{"description":"Requests remaining in the active rate-limit window.","schema":{"type":"string"}},"X-RateLimit-Reset":{"description":"Unix epoch milliseconds when the active rate-limit window resets.","schema":{"type":"string"}}}},"default":{"content":{"application/json":{"schema":{"oneOf":[{"$ref":"#/components/schemas/Error"},{"$ref":"#/components/schemas/GatewayError"}]}}},"description":"Gateway or handler error response."}},"summary":"SummarizeArticle","tags":["NewsService"]}},"/api/news/v1/summarize-article-cache":{"get":{"description":"GetSummarizeArticleCache looks up a cached summary by deterministic key (CDN-cacheable GET).","operationId":"GetSummarizeArticleCache","parameters":[{"description":"Deterministic cache key computed by buildSummaryCacheKey().","example":"summary:v1:example-cache","in":"query","name":"cache_key","required":false,"schema":{"pattern":"^summary:v\\d+:[a-z0-9:_-]{3,120}$","type":"string"}},{"description":"Optional JMESPath expression applied server-side to project or reduce the JSON response before it is returned (mirrors the MCP jmespath argument). Invalid expressions, expressions larger than 1024 UTF-8 bytes, or projections that exceed the 256 KB output cap return HTTP 400 with a {_jmespath_error, original_keys} envelope. Grammar and worked examples: https://www.worldmonitor.app/docs/mcp-jmespath.","example":"keys(@)","in":"query","name":"jmespath","required":false,"schema":{"type":"string"}}],"responses":{"200":{"content":{"application/json":{"example":{"error":"example","errorType":"all","fallback":true,"model":"example","provider":"openrouter"},"schema":{"$ref":"#/components/schemas/SummarizeArticleResponse"}}},"description":"Successful response"},"400":{"content":{"application/json":{"schema":{"oneOf":[{"$ref":"#/components/schemas/ValidationError"},{"$ref":"#/components/schemas/JmespathProjectionError"}]}}},"description":"Validation error"},"401":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UnauthorizedError"}}},"description":"Missing or invalid API key."},"403":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ForbiddenError"}}},"description":"API access requires an active subscription (the API key's subscription is inactive or expired)."},"429":{"content":{"application/json":{"schema":{"oneOf":[{"$ref":"#/components/schemas/Error"},{"$ref":"#/components/schemas/RateLimitError"}]}}},"description":"Rate limit exceeded.","headers":{"Retry-After":{"description":"Seconds to wait before retrying the request.","schema":{"type":"string"}},"X-RateLimit-Limit":{"description":"Maximum requests allowed in the active rate-limit window.","schema":{"type":"string"}},"X-RateLimit-Remaining":{"description":"Requests remaining in the active rate-limit window.","schema":{"type":"string"}},"X-RateLimit-Reset":{"description":"Unix epoch milliseconds when the active rate-limit window resets.","schema":{"type":"string"}}}},"default":{"content":{"application/json":{"schema":{"oneOf":[{"$ref":"#/components/schemas/Error"},{"$ref":"#/components/schemas/GatewayError"}]}}},"description":"Gateway or handler error response."}},"summary":"GetSummarizeArticleCache","tags":["NewsService"]}}},"security":[{"WorldMonitorKey":[]},{"ApiKeyHeader":[]}],"servers":[{"url":"https://api.worldmonitor.app"}]} \ No newline at end of file diff --git a/docs/api/NewsService.openapi.yaml b/docs/api/NewsService.openapi.yaml index 4c2a45eacc..18dd32484c 100644 --- a/docs/api/NewsService.openapi.yaml +++ b/docs/api/NewsService.openapi.yaml @@ -662,7 +662,12 @@ components: corroborationCount: type: integer format: int32 - description: Number of distinct sources that reported the same story in this digest cycle. + description: |- + Number of distinct sources that reported the same story in this digest + cycle. "Same story" is determined by edit-tolerant story-identity + clustering (headlines describing one event with different wording, + ordering, source suffixes, truncation, or morphology corroborate each + other) — not exact-title matching. See shared/story-identity.js. storyMeta: $ref: '#/components/schemas/StoryMeta' snippet: diff --git a/docs/api/worldmonitor.openapi.yaml b/docs/api/worldmonitor.openapi.yaml index 98d55d3e28..e48e5f9e3c 100644 --- a/docs/api/worldmonitor.openapi.yaml +++ b/docs/api/worldmonitor.openapi.yaml @@ -32151,7 +32151,12 @@ components: corroborationCount: type: integer format: int32 - description: Number of distinct sources that reported the same story in this digest cycle. + description: |- + Number of distinct sources that reported the same story in this digest + cycle. "Same story" is determined by edit-tolerant story-identity + clustering (headlines describing one event with different wording, + ordering, source suffixes, truncation, or morphology corroborate each + other) — not exact-title matching. See shared/story-identity.js. storyMeta: $ref: '#/components/schemas/worldmonitor_news_v1_StoryMeta' snippet: diff --git a/proto/worldmonitor/news/v1/news_item.proto b/proto/worldmonitor/news/v1/news_item.proto index 5cb4f35c2d..19dfb378dc 100644 --- a/proto/worldmonitor/news/v1/news_item.proto +++ b/proto/worldmonitor/news/v1/news_item.proto @@ -36,7 +36,11 @@ message NewsItem { // corroborating source, capped at five sources. The final score can exceed // 100; with current boosts it is approximately capped at 138. int32 importance_score = 9; - // Number of distinct sources that reported the same story in this digest cycle. + // Number of distinct sources that reported the same story in this digest + // cycle. "Same story" is determined by edit-tolerant story-identity + // clustering (headlines describing one event with different wording, + // ordering, source suffixes, truncation, or morphology corroborate each + // other) — not exact-title matching. See shared/story-identity.js. int32 corroboration_count = 10; // Story lifecycle metadata derived from cross-cycle persistence data. StoryMeta story_meta = 11; diff --git a/scripts/_clustering.mjs b/scripts/_clustering.mjs index ffb0ba9315..0ac9ee48ae 100644 --- a/scripts/_clustering.mjs +++ b/scripts/_clustering.mjs @@ -1,6 +1,12 @@ #!/usr/bin/env node import { createRequire } from 'node:module'; +// #4919: story similarity is delegated to the shared story-identity +// module (scripts/shared mirror — same rootDirectory=scripts reason as +// the JSON requires below). The local Jaccard-0.5 matcher this file +// carried was one of three inconsistent "same story?" answers in the +// codebase; all three now share ONE definition and threshold. +import { clusterTexts } from './shared/story-identity.js'; const require = createRequire(import.meta.url); const SOURCE_TIERS = require('./shared/source-tiers.json'); @@ -9,20 +15,8 @@ const SOURCE_TIERS = require('./shared/source-tiers.json'); // is not in the container. Matches the SOURCE_TIERS pattern above. const DIPLOMACY_KEYWORDS_DATA = require('./shared/diplomacy-keywords.json'); -const SIMILARITY_THRESHOLD = 0.5; const ENTITY_CORROBORATION_WINDOW_MS = 24 * 60 * 60 * 1000; -const STOP_WORDS = new Set([ - 'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', - 'of', 'with', 'by', 'from', 'as', 'is', 'was', 'are', 'were', 'been', - 'be', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', - 'could', 'should', 'may', 'might', 'must', 'shall', 'can', 'need', - 'it', 'its', 'this', 'that', 'these', 'those', 'i', 'you', 'he', - 'she', 'we', 'they', 'what', 'which', 'who', 'whom', 'how', 'when', - 'where', 'why', 'all', 'each', 'every', 'both', 'few', 'more', 'most', - 'other', 'some', 'such', 'no', 'not', 'only', 'same', 'so', 'than', - 'too', 'very', 'just', 'also', 'now', 'new', 'says', 'said', 'after', -]); const MILITARY_KEYWORDS = [ 'war', 'armada', 'invasion', 'airstrike', 'strike', 'missile', 'troops', @@ -57,14 +51,6 @@ const DEMOTE_KEYWORDS = [ 'quarterly', 'profit', 'investor', 'ipo', 'funding', 'valuation', ]; -function tokenize(text) { - const words = text - .toLowerCase() - .replace(/[^a-z0-9\s]/g, ' ') - .split(/\s+/) - .filter(w => w.length > 2 && !STOP_WORDS.has(w)); - return new Set(words); -} function finiteNumber(value, fallback = 0) { const n = Number(value); @@ -131,59 +117,15 @@ function containsKeywordToken(text, kw) { return new RegExp(`(^|\\s)${escaped}`).test(text); } -function jaccardSimilarity(a, b) { - if (a.size === 0 && b.size === 0) return 0; - let intersection = 0; - for (const x of a) { - if (b.has(x)) intersection++; - } - const union = a.size + b.size - intersection; - return intersection / union; -} - export function clusterItems(items) { if (items.length === 0) return []; - const tokenList = items.map(item => tokenize(item.title || '')); - - const invertedIndex = new Map(); - for (let i = 0; i < tokenList.length; i++) { - for (const token of tokenList[i]) { - const bucket = invertedIndex.get(token); - if (bucket) bucket.push(i); - else invertedIndex.set(token, [i]); - } - } - - const clusters = []; - const assigned = new Set(); - - for (let i = 0; i < items.length; i++) { - if (assigned.has(i)) continue; - - const cluster = [i]; - assigned.add(i); - const tokensI = tokenList[i]; - - const candidates = new Set(); - for (const token of tokensI) { - const bucket = invertedIndex.get(token); - if (!bucket) continue; - for (const idx of bucket) { - if (idx > i) candidates.add(idx); - } - } - - for (const j of Array.from(candidates).sort((a, b) => a - b)) { - if (assigned.has(j)) continue; - if (jaccardSimilarity(tokensI, tokenList[j]) >= SIMILARITY_THRESHOLD) { - cluster.push(j); - assigned.add(j); - } - } - - clusters.push(cluster.map(idx => items[idx])); - } + // #4924 review (maintainability P1): delegate the clustering ALGORITHM + // to the shared module too, not just the similarity function — a local + // copy of the loop would let clustering semantics drift apart again, + // one layer above the drift this PR removed. + const clusters = clusterTexts(items.map(item => item.title || '')) + .map(indices => indices.map(idx => items[idx])); return clusters.map(group => { const sorted = [...group].sort((a, b) => { diff --git a/scripts/shared/story-identity.js b/scripts/shared/story-identity.js new file mode 100644 index 0000000000..b77769e0f2 --- /dev/null +++ b/scripts/shared/story-identity.js @@ -0,0 +1,443 @@ +/** + * story-identity — the ONE similarity definition for "are these the same + * news story?" (#4919, Bet 1 of the 2026-07-05 strategic review). + * + * Before this module the codebase answered that question three different + * ways: scripts/_clustering.mjs (title-token Jaccard ≥ 0.5), + * server/worldmonitor/news/v1/dedup.mjs (word-overlap > 0.6 of the smaller + * set), and list-feed-digest.ts story tracking (EXACT sha256 of the + * normalized title — so any wording edit forked the story and deflated + * corroboration). All three now delegate here. + * + * ── Method ────────────────────────────────────────────────────────────── + * DUAL-VIEW feature-hashed lexical vectors; similarity = min of the two + * views' cosines (see lexicalStoryVector for why two views). Features: + * - word tokens (weight 2.0) — core lexical identity + * - word bigrams (weight 1.5) — order/direction ("ukraine + * drone" vs "russian drone" separates actor-flipped headlines that + * bag-of-words alone cannot) + * - char 4-grams per token (weight 1.0) — morphology fuzz + * (iran/iranian, sanction/sanctions) + * - char bigrams for non-ASCII tokens — CJK and other unsegmented + * scripts get no whitespace tokens, so bigrams carry the signal + * hashed (signed FNV-1a) into 512 dims and L2-normalized. Deterministic, + * dependency-free, script-agnostic, ~µs per title. + * + * This is an EDIT-TOLERANT identity, not a semantic one: it merges the + * real-world corroboration killers (source suffixes, truncations, + * qualifier edits, reorders, morphology) and keeps distinct events apart. + * It can NOT merge a full cross-language paraphrase ("Iran threatens…" / + * "Teherán amenaza…") — that requires a semantic embedding provider, + * which plugs in behind `setStoryVectorProvider()` without touching any + * consumer. Known hard limit either way: two events differing by one + * token ("12th sanctions package" vs "13th sanctions package") are not + * separable by similarity alone; the 96h ingest window bounds the damage. + * + * Mirrored byte-for-byte to scripts/shared/story-identity.js (enforced by + * tests/scripts-shared-mirror.test.mjs — Railway seed bundles deploy with + * rootDirectory=scripts and cannot see repo-root shared/). + */ + +const DIM = 512; + +// Tuned on the labeled pair set in tests/story-identity.test.mjs +// (edit-variant positives vs same-topic-different-event negatives). The +// test asserts full separation with margin on both sides; retune there +// if the vectorizer changes. +export const STORY_SIMILARITY_THRESHOLD = 0.615; + +const WEIGHT_TOKEN = 2.0; +const WEIGHT_BIGRAM = 1.5; +const WEIGHT_CHARGRAM = 1.0; +// Discriminative boosts, applied to the token feature only (bigrams keep +// their flat weight — they already encode order). Without these, a +// one-entity swap ("Turkey hikes rates to 50%" vs "Argentina hikes rates +// to 50%") scores ~0.82 because the shared verb/number mass dominates; +// capitalized-in-raw-text tokens are entity-shaped and numbers are +// event parameters (magnitudes, percentages, ordinals), so both carry +// the discriminating signal. In Title Case or ALL-CAPS headlines every +// token gets the boost — uniform scaling, which cosine ignores — so the +// heuristic only sharpens sentence-case headlines and never hurts. +const BOOST_ENTITY = 3.0; +const BOOST_NUMBER = 2.0; + +/** FNV-1a 32-bit over a string, with a seed so we can derive two + * independent hashes (index + sign) from one feature. */ +function fnv1a(str, seed) { + let h = (0x811c9dc5 ^ seed) >>> 0; + for (let i = 0; i < str.length; i++) { + h ^= str.charCodeAt(i); + h = Math.imul(h, 0x01000193) >>> 0; + } + return h >>> 0; +} + +/** + * Generic story-text normalization: lowercase, strip everything that is + * not a Unicode letter/number, collapse whitespace. Callers that know + * about source-attribution suffixes ("… - Reuters") strip those BEFORE + * calling (list-feed-digest's normalizeTitle already does). + * @param {string} text + * @returns {string} + */ +export function normalizeStoryText(text) { + return (text || '') + .toLowerCase() + .replace(/[^\p{L}\p{N}\s]/gu, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +/** @param {string} token */ +function isNonAscii(token) { + for (let i = 0; i < token.length; i++) { + if (token.charCodeAt(i) > 127) return true; + } + return false; +} + +/** + * Tokens used for inverted-index candidate generation by clustering + * callers (cheap pre-filter: only pairs sharing ≥1 token are scored). + * ASCII tokens shorter than 3 chars are dropped (stopword-weight noise); + * non-ASCII tokens are kept whole AND as char bigrams so unsegmented + * scripts still produce index keys. + * @param {string} text + * @returns {Set} + */ +export function candidateTokens(text) { + const out = new Set(); + const clamped = stripAttributionSuffix(text).slice(0, MAX_IDENTITY_CHARS); + for (const tok of normalizeStoryText(clamped).split(' ')) { + if (!tok) continue; + if (isNonAscii(tok)) { + out.add(tok); + for (let i = 0; i + 2 <= tok.length; i++) out.add(tok.slice(i, i + 2)); + } else if (tok.length >= 3) { + out.add(tok); + } + } + return out; +} + +// Trailing source-attribution suffixes ("… - Reuters", "… - example.com") +// must not enter the vector: Google-News wrapper titles carry them on +// EVERY item, so the publisher token (capitalized → entity-boosted ×3) +// adds shared mass across DISTINCT same-publisher stories and pulls them +// toward a false merge (cross-model review finding, PR #4924). Mirrors +// list-feed-digest's normalizeTitle suffix rules, but case-preserving. +const ATTRIBUTION_SUFFIX_RES = [ + /\s*[-\u2013\u2014|]\s*[\w\s.]+\.(?:com|org|net|co\.uk)\s*$/i, + /\s*[-\u2013\u2014|]\s*(?:reuters|ap news|bbc|cnn|al jazeera|france 24|dw news|pbs newshour|cbs news|nbc|abc|associated press|the guardian|nos nieuws|tagesschau|cnbc|the national)\s*$/i, +]; + +// Unbounded feed titles feed char-4gram loops inside a 25s serverless +// budget; clamp AFTER suffix stripping. 300 chars ≈ 3× a long headline. +const MAX_IDENTITY_CHARS = 300; + +/** @param {string} text @returns {string} */ +export function stripAttributionSuffix(text) { + let out = text || ''; + for (const re of ATTRIBUTION_SUFFIX_RES) out = out.replace(re, ''); + return out; +} + +/** + * Content tokens WITH the discriminative flags read from the raw + * (pre-lowercase) text. Callers should pass raw titles — lowercasing + * upstream destroys the entity signal (harmless, but the boost is lost). + * @param {string} text + * @returns {Array<{ tok: string; boost: number }>} + */ +function contentTokens(text) { + const kept = []; + const clamped = stripAttributionSuffix(text).slice(0, MAX_IDENTITY_CHARS); + for (const raw of clamped.split(/\s+/)) { + // Strip everything that is not a Unicode letter/number, keeping the + // original case so the entity heuristic can read it. + const clean = raw.replace(/[^\p{L}\p{N}]/gu, ''); + if (!clean) continue; + const tok = clean.toLowerCase(); + if (!isNonAscii(tok) && tok.length < 3) continue; + const capitalized = /^\p{Lu}/u.test(clean); + const hasDigit = /\p{N}/u.test(clean); + const boost = hasDigit ? BOOST_NUMBER : capitalized ? BOOST_ENTITY : 1; + kept.push({ tok, boost }); + } + return kept; +} + +/** @param {Float64Array} vec @param {string} feature @param {number} weight */ +function addFeature(vec, feature, weight) { + const idx = fnv1a(feature, 0) % DIM; + const sign = (fnv1a(feature, 0x9e3779b9) & 1) === 1 ? 1 : -1; + vec[idx] += sign * weight; +} + +/** @param {Float64Array} vec */ +function l2normalize(vec) { + let norm = 0; + for (let i = 0; i < DIM; i++) norm += vec[i] * vec[i]; + norm = Math.sqrt(norm); + if (norm === 0) return null; + for (let i = 0; i < DIM; i++) vec[i] /= norm; + return vec; +} + +/** + * The default lexical vectorizer — DUAL VIEW. Returns two L2-normalized + * 512-dim views of the same text: + * - `u` (uniform): every token feature at flat weight. Sensitive to + * action/verb substitutions ("seizes tanker" vs "threatens to + * close") that entity weighting would wash out. + * - `b` (boosted): entity-shaped (capitalized-in-raw) tokens ×3 and + * numeric tokens ×2. Sensitive to one-entity swaps ("Turkey hikes + * rates…" vs "Argentina hikes rates…") that flat weighting scores + * ~0.82 because the shared verb mass dominates. + * A pair is the same story only when BOTH views agree (similarity = + * min of the two cosines) — each view catches the failure mode the + * other is blind to. Tuned on the labeled pair set in + * tests/story-identity.test.mjs: min positive 0.634, max negative + * 0.595 with THRESHOLD 0.615 between them. + * + * Returns null for texts with no usable tokens (callers treat null as + * "cannot match" — never same-story). + * @param {string} text + * @returns {{ u: Float64Array; b: Float64Array } | null} + */ +function lexicalStoryVector(text) { + const tokens = contentTokens(text); + if (tokens.length === 0) return null; + const u = new Float64Array(DIM); + const b = new Float64Array(DIM); + for (let i = 0; i < tokens.length; i++) { + const { tok, boost } = tokens[i]; + addFeature(u, `w:${tok}`, WEIGHT_TOKEN); + addFeature(b, `w:${tok}`, WEIGHT_TOKEN * boost); + if (i + 1 < tokens.length) { + const bigram = `b:${tok} ${tokens[i + 1].tok}`; + addFeature(u, bigram, WEIGHT_BIGRAM); + addFeature(b, bigram, WEIGHT_BIGRAM); + } + if (isNonAscii(tok)) { + // Unsegmented-script fallback: char bigrams of the raw token. + for (let j = 0; j + 2 <= tok.length; j++) { + const g = `c2:${tok.slice(j, j + 2)}`; + addFeature(u, g, WEIGHT_CHARGRAM); + addFeature(b, g, WEIGHT_CHARGRAM); + } + } + if (tok.length >= 4) { + const padded = `<${tok}>`; + for (let j = 0; j + 4 <= padded.length; j++) { + const g = `c4:${padded.slice(j, j + 4)}`; + addFeature(u, g, WEIGHT_CHARGRAM); + addFeature(b, g, WEIGHT_CHARGRAM); + } + } + } + const un = l2normalize(u); + const bn = l2normalize(b); + if (!un || !bn) return null; + // Token set rides along for the containment rescue in + // cosineSimilarity — severe RSS truncation (a headline cut to ~40% of + // its tokens) drops the cosine below threshold even though the short + // form is a strict subset of the long form. The old word-overlap + // dedup metric (|∩|/min) handled exactly this class; keep that + // guarantee via token containment. + return { u: un, b: bn, t: new Set(tokens.map((entry) => entry.tok)) }; +} + +/** Active vectorizer — swappable for a semantic embedding provider. */ +let activeVectorizer = lexicalStoryVector; + +/** + * Plug in a semantic embedding provider (must be synchronous or the + * caller precomputes; must return `{ u, b }` of L2-normalized + * Float64Arrays of a consistent dimension — a single-embedding provider + * sets u === b — or null). Pass null to restore the default lexical + * vectorizer. Consumers never change — only the vector source. + * @param {((text: string) => { u: Float64Array; b: Float64Array } | null) | null} provider + */ +export function setStoryVectorProvider(provider) { + activeVectorizer = typeof provider === 'function' ? provider : lexicalStoryVector; +} + +/** + * @param {string} text + * @returns {{ u: Float64Array; b: Float64Array } | null} dual-view story + * vector (opaque — pass to cosineSimilarity), or null when the text + * has no usable signal. + */ +export function storyVector(text) { + return activeVectorizer(text); +} + +/** @param {Float64Array} a @param {Float64Array} b */ +function dot(a, b) { + if (a.length !== b.length) return 0; + let d = 0; + for (let i = 0; i < a.length; i++) d += a[i] * b[i]; + return d; +} + +/** + * Similarity of two dual-view story vectors: the MIN of the uniform-view + * and boosted-view cosines — a pair is the same story only when both + * views agree. Null vectors never match anything. + * @param {{ u: Float64Array; b: Float64Array } | null} a + * @param {{ u: Float64Array; b: Float64Array } | null} b + * @returns {number} + */ +// Containment rescue floor: a title whose content tokens are ≥90% +// contained in the other's (with at least 4 tokens on the smaller side, +// so fragments like "Iran" can't rescue) IS the same story — the +// truncated-wire-copy class the old |∩|/min dedup metric guaranteed. +const CONTAINMENT_RESCUE_MIN_TOKENS = 4; +const CONTAINMENT_RESCUE_RATIO = 0.9; +const CONTAINMENT_RESCUE_SCORE = 0.9; + +export function cosineSimilarity(a, b) { + if (!a || !b) return 0; + const score = Math.min(dot(a.u, b.u), dot(a.b, b.b)); + // Rescue only applies to lexical vectors carrying token sets — a + // semantic provider's vectors skip it (semantic cosine already + // handles truncation). + if (score < CONTAINMENT_RESCUE_SCORE && a.t && b.t) { + const [small, large] = a.t.size <= b.t.size ? [a.t, b.t] : [b.t, a.t]; + if (small.size >= CONTAINMENT_RESCUE_MIN_TOKENS) { + let shared = 0; + for (const tok of small) { + if (large.has(tok)) shared++; + } + if (shared / small.size >= CONTAINMENT_RESCUE_RATIO) { + return CONTAINMENT_RESCUE_SCORE; + } + } + } + return score; +} + +/** + * Convenience: similarity of two raw texts. + * @param {string} textA @param {string} textB + * @returns {number} + */ +export function storySimilarity(textA, textB) { + return cosineSimilarity(storyVector(textA), storyVector(textB)); +} + +/** + * @param {string} textA @param {string} textB + * @param {number} [threshold] + * @returns {boolean} + */ +export function isSameStory(textA, textB, threshold = STORY_SIMILARITY_THRESHOLD) { + return storySimilarity(textA, textB) >= threshold; +} + +// A token shared by more than this many titles carries no clustering +// signal (it is the batch's "the") but drives O(bucket²) pair scoring — +// an adversarial or organic hot-entity spike (thousands of titles naming +// one entity) would otherwise burn seconds of CPU inside the digest +// handler's 25s budget. Pairs joined ONLY by ultra-hot tokens almost +// always share a rarer token too. +const MAX_CANDIDATE_BUCKET = 250; + +/** + * Cluster texts into same-story groups: connected components over the + * "similarity ≥ threshold" edge set (union-find), with inverted-index + * candidate generation so only pairs sharing ≥1 token are scored. + * + * Connected components — NOT the greedy first-seed pass the legacy + * _clustering.mjs used — because component membership is independent of + * input order: feed arrival order varies run to run, and under greedy + * assignment a chain (A~B, B~C, A≁C) could land C in or out of A's + * cluster depending on which seeded first, churning the canonical + * story:track identity downstream (cross-model review finding, + * PR #4924). Transitive chains merge by design; the threshold's + * precision bounds chain length in practice. + * @param {string[]} texts + * @param {{ threshold?: number }} [opts] + * @returns {number[][]} clusters of indices into `texts`, ordered by + * smallest member index; members ascending + */ +export function clusterTexts(texts, opts = {}) { + const threshold = typeof opts.threshold === 'number' ? opts.threshold : STORY_SIMILARITY_THRESHOLD; + const vectors = texts.map((t) => storyVector(t)); + const tokenSets = texts.map((t) => candidateTokens(t)); + + // Exact-duplicate pre-union (#4924 external review): identical + // normalized texts union unconditionally BEFORE the candidate scan. + // Without this, a mega-story (e.g. 251 verbatim syndications) makes + // every shared token bucket exceed MAX_CANDIDATE_BUCKET, no pairs get + // scored, and the most-corroborated story of the day degrades to + // singletons — the exact case corroboration exists for. + const byExactText = new Map(); + + const invertedIndex = new Map(); + for (let i = 0; i < texts.length; i++) { + for (const token of tokenSets[i]) { + const bucket = invertedIndex.get(token); + if (bucket) bucket.push(i); + else invertedIndex.set(token, [i]); + } + } + + const parent = new Array(texts.length); + for (let i = 0; i < texts.length; i++) parent[i] = i; + + const find = (x) => { + let root = x; + while (parent[root] !== root) root = parent[root]; + while (parent[x] !== root) { + const next = parent[x]; + parent[x] = root; + x = next; + } + return root; + }; + const union = (a, b) => { + const ra = find(a); + const rb = find(b); + if (ra === rb) return; + // Deterministic: smaller index becomes the root. + if (ra < rb) parent[rb] = ra; + else parent[ra] = rb; + }; + + for (let i = 0; i < texts.length; i++) { + const normalized = normalizeStoryText(texts[i]); + if (!normalized) continue; + const first = byExactText.get(normalized); + if (first === undefined) byExactText.set(normalized, i); + else union(first, i); + } + + for (let i = 0; i < texts.length; i++) { + if (!vectors[i]) continue; + const candidates = new Set(); + for (const token of tokenSets[i]) { + const bucket = invertedIndex.get(token); + if (!bucket || bucket.length > MAX_CANDIDATE_BUCKET) continue; + for (const idx of bucket) { + if (idx > i) candidates.add(idx); + } + } + for (const j of candidates) { + if (find(i) === find(j)) continue; + if (cosineSimilarity(vectors[i], vectors[j]) >= threshold) union(i, j); + } + } + + const byRoot = new Map(); + for (let i = 0; i < texts.length; i++) { + const root = find(i); + const members = byRoot.get(root); + if (members) members.push(i); + else byRoot.set(root, [i]); + } + return Array.from(byRoot.entries()) + .sort((a, b) => a[0] - b[0]) + .map(([, members]) => members); +} diff --git a/server/_shared/cache-keys.ts b/server/_shared/cache-keys.ts index 3d9cf9ef35..ae97f9ee50 100644 --- a/server/_shared/cache-keys.ts +++ b/server/_shared/cache-keys.ts @@ -44,6 +44,10 @@ export const DIGEST_ACCUMULATOR_KEY_PREFIX = 'digest:accumulator:v1:'; export const STORY_TRACK_KEY = (titleHash: string) => `story:track:v1:${titleHash}`; export const STORY_SOURCES_KEY = (titleHash: string) => `story:sources:v1:${titleHash}`; export const STORY_PEAK_KEY = (titleHash: string) => `story:peak:v1:${titleHash}`; +// #4924: member exact-title hash -> canonical story hash, same TTL as the +// track — lets a later cycle adopt the live canonical when the original +// canonical member is absent from the batch. +export const STORY_ALIAS_KEY = (titleHash: string) => `story:alias:v1:${titleHash}`; export const DIGEST_ACCUMULATOR_KEY = (variant: string, lang = 'en') => `digest:accumulator:v1:${variant}:${lang}`; export const DIGEST_LAST_SENT_KEY = (userId: string, variant: string) => `digest:last-sent:v1:${userId}:${variant}`; // NOTE: notification-relay.cjs owns the live value (shadow:score-log:v5 since prompt upgrade). diff --git a/server/worldmonitor/news/v1/_shared.ts b/server/worldmonitor/news/v1/_shared.ts index 8f3fec1997..50fdf90406 100644 --- a/server/worldmonitor/news/v1/_shared.ts +++ b/server/worldmonitor/news/v1/_shared.ts @@ -26,7 +26,6 @@ export { hashString }; // Headline deduplication (used by SummarizeArticle) // ======================================================================== -// @ts-expect-error -- plain JS module, no .d.mts needed for this pure function export { deduplicateHeadlines } from './dedup.mjs'; // ======================================================================== diff --git a/server/worldmonitor/news/v1/dedup.d.mts b/server/worldmonitor/news/v1/dedup.d.mts new file mode 100644 index 0000000000..739158f453 --- /dev/null +++ b/server/worldmonitor/news/v1/dedup.d.mts @@ -0,0 +1,15 @@ +/** Types for dedup.mjs (plain JS so .mjs tests can import it directly). */ + +export function deduplicateHeadlines(headlines: string[]): string[]; + +export function assignStoryIdentity( + items: T[], + normalizeTitle: (title: string) => string, + sha256Hex: (text: string) => Promise, +): Promise>; + +export function adoptExistingCanonical( + memberTitleHashes: string[] | undefined, + defaultHash: string, + aliasTargetByHash: Map | Record | undefined, +): string; diff --git a/server/worldmonitor/news/v1/dedup.mjs b/server/worldmonitor/news/v1/dedup.mjs index cea72a6c59..f65e20957e 100644 --- a/server/worldmonitor/news/v1/dedup.mjs +++ b/server/worldmonitor/news/v1/dedup.mjs @@ -1,29 +1,156 @@ /** - * Headline deduplication using word-level similarity. + * Headline dedup + story-identity assignment for the news pipeline. * Plain JS module so it can be imported from both TS source and .mjs tests. + * + * #4919: similarity is delegated to shared/story-identity.js — the single + * "same news story?" definition (previously this file carried its own + * word-overlap>0.6 matcher, one of three inconsistent answers in the + * codebase). */ +import { + storyVector, + cosineSimilarity, + clusterTexts, + STORY_SIMILARITY_THRESHOLD, +} from '../../../../shared/story-identity.js'; + /** @param {string[]} headlines */ export function deduplicateHeadlines(headlines) { - const seen = []; + const seenVectors = []; const unique = []; - for (const headline of headlines) { - const normalized = headline.toLowerCase().replace(/[^\w\s]/g, '').replace(/\s+/g, ' ').trim(); - const words = new Set(normalized.split(' ').filter((w) => w.length >= 4)); - - let isDuplicate = false; - for (const seenWords of seen) { - const intersection = [...words].filter((w) => seenWords.has(w)); - const similarity = intersection.length / Math.min(words.size, seenWords.size); - if (similarity > 0.6) { isDuplicate = true; break; } - } - + const vec = storyVector(headline); + // Unvectorizable headlines (empty/punctuation-only) can't be compared; + // keep them rather than silently dropping content. + const isDuplicate = vec !== null + && seenVectors.some((seen) => cosineSimilarity(vec, seen) >= STORY_SIMILARITY_THRESHOLD); if (!isDuplicate) { - seen.push(words); + if (vec !== null) seenVectors.push(vec); unique.push(headline); } } - return unique; } + +/** + * Cluster a request batch of feed items into stories and assign each item + * its cluster's canonical identity (#4919). Replaces the exact + * sha256(normalizeTitle) identity that forked a story on ANY wording edit + * and deflated corroboration to verbatim-syndication-only. + * + * Canonical id = hash of the normalized title of the EARLIEST-published + * member (ties and missing timestamps fall back to the lexicographically + * smallest normalized title). Anchoring on the oldest member keeps the + * story:track identity stable while a story actively corroborates: a + * newly-joining wording, by definition, publishes later and therefore can + * never steal the canonical mid-lifecycle (PR #4924 review — reliability + * P1 + the 3-incident story:track continuity history in + * list-feed-digest.ts). The id changes only when the oldest member ages + * out of the 96h window — the same orphaning every wording variant + * suffered under exact hashing, so worst case equals old behavior. + * (Residual, tracked as follow-up: a hostile feed can still backdate its + * publishedAt within the freshness window to claim the canonical — + * requires the cross-cycle adopt-existing-track hardening.) + * + * Items whose normalized title is EMPTY (emoji/punctuation-only) get a + * per-item sentinel identity instead of sharing sha256("") — under exact + * hashing all such items accumulated one phantom story:track row with + * pooled corroboration. + * + * @template {{ title: string; source: string; publishedAt?: number }} T + * @param {T[]} items + * @param {(title: string) => string} normalizeTitle title normalizer + * (strips source suffixes etc. — stays caller-owned so hash identity is + * unchanged for singleton clusters) + * @param {(text: string) => Promise} sha256Hex + * @returns {Promise>} + */ +export async function assignStoryIdentity(items, normalizeTitle, sha256Hex) { + const clusters = clusterTexts(items.map((item) => item.title || '')); + const assignment = new Map(); + await Promise.all(clusters.map(async (indices) => { + let canonical = null; + let canonicalPublishedAt = Infinity; + for (const i of indices) { + const normalized = normalizeTitle(items[i].title || ''); + if (!normalized) continue; + const publishedAt = typeof items[i].publishedAt === 'number' && Number.isFinite(items[i].publishedAt) + ? items[i].publishedAt + : Infinity; + if ( + canonical === null + || publishedAt < canonicalPublishedAt + || (publishedAt === canonicalPublishedAt && normalized < canonical) + ) { + canonical = normalized; + canonicalPublishedAt = publishedAt; + } + } + + const sources = new Set(); + for (const i of indices) { + if (items[i].source) sources.add(items[i].source); + } + const corroborationCount = Math.max(1, sources.size); + + if (canonical === null) { + // Whole cluster normalizes to empty — sentinel identity per item, + // no shared phantom track, no pooled corroboration. + await Promise.all(indices.map(async (i) => { + const titleHash = await sha256Hex(`untrackable:${items[i].source || ''}:${items[i].title || ''}`); + // No alias rows for sentinel identities — they are per-item by design. + assignment.set(items[i], { titleHash, corroborationCount: 1, memberTitleHashes: [] }); + })); + return; + } + + const titleHash = await sha256Hex(canonical); + // Unique exact-title hashes of every member — the caller persists + // memberHash->canonicalHash alias rows and adopts a live canonical on + // later cycles (#4924 review P1: without this, cycle 1 = A+B with A + // canonical wrote only A's track; a cycle-2 B-only batch hashed to B + // and RESET the story to BREAKING/mentionCount=1 — worse than the old + // exact hashing, where B's own track would have continued). + const memberNormalized = new Set(); + for (const i of indices) { + const normalized = normalizeTitle(items[i].title || ''); + if (normalized) memberNormalized.add(normalized); + } + const memberTitleHashes = await Promise.all([...memberNormalized].map((n) => sha256Hex(n))); + for (const i of indices) { + assignment.set(items[i], { titleHash, corroborationCount, memberTitleHashes }); + } + })); + return assignment; +} + +/** + * Pure canonical-adoption rule (#4924 review P1). Given a cluster's member + * exact-title hashes, the batch-derived default canonical hash, and a map of + * live alias rows (memberHash -> canonical hash written in a previous + * cycle, same TTL as story tracks), return the hash the cluster should + * track under: the live canonical most of its members already point at — + * so a story keeps its identity when the original canonical member drops + * out of the batch. Deterministic: most-common target wins, ties break to + * the lexicographically smallest hash. No live alias -> default. + */ +export function adoptExistingCanonical(memberTitleHashes, defaultHash, aliasTargetByHash) { + const counts = new Map(); + for (const memberHash of Array.isArray(memberTitleHashes) ? memberTitleHashes : []) { + const target = aliasTargetByHash instanceof Map + ? aliasTargetByHash.get(memberHash) + : aliasTargetByHash?.[memberHash]; + if (typeof target !== 'string' || target.length === 0) continue; + counts.set(target, (counts.get(target) ?? 0) + 1); + } + let adopted = null; + let best = 0; + for (const [target, count] of counts) { + if (count > best || (count === best && adopted !== null && target < adopted)) { + adopted = target; + best = count; + } + } + return adopted ?? defaultHash; +} diff --git a/server/worldmonitor/news/v1/list-feed-digest.ts b/server/worldmonitor/news/v1/list-feed-digest.ts index e065f10b9d..ce973dd0b6 100644 --- a/server/worldmonitor/news/v1/list-feed-digest.ts +++ b/server/worldmonitor/news/v1/list-feed-digest.ts @@ -14,6 +14,7 @@ import { sha256Hex } from '../../../_shared/hash'; import { CHROME_UA } from '../../../_shared/constants'; import { VARIANT_FEEDS, INTEL_SOURCES, type ServerFeed } from './_feeds'; import { classifyByKeyword, hasHistoricalMarker, type ThreatLevel } from './_classifier'; +import { assignStoryIdentity, adoptExistingCanonical } from './dedup.mjs'; import { classifyOpinion } from '../../../_shared/opinion-classifier.js'; import { classifyFeelGood } from '../../../_shared/feelgood-classifier.js'; import { classifyEphemeralLiveCoverage } from '../../../../shared/ephemeral-live-classifier.js'; @@ -23,6 +24,7 @@ import { STORY_TRACK_KEY, STORY_SOURCES_KEY, STORY_PEAK_KEY, + STORY_ALIAS_KEY, DIGEST_ACCUMULATOR_KEY, STORY_TTL, STORY_TRACK_KEY_PREFIX, @@ -1109,11 +1111,39 @@ function buildStoryTrackHsetFields( ]; } -async function writeStoryTracking(items: ParsedItem[], variant: string, lang: string, hashes: string[]): Promise { +async function writeStoryTracking(items: ParsedItem[], variant: string, lang: string, hashes: string[], memberHashesByFinal?: Map>): Promise { if (items.length === 0) return; const now = Date.now(); const accKey = DIGEST_ACCUMULATOR_KEY(variant, lang); + // #4919/#4924: with fuzzy story identity, N same-cycle wording variants + // share one titleHash. Mutable per-story writes (mentionCount HINCRBY, + // HSET representative fields) must run ONCE per unique hash per cycle — + // per-item they would inflate mentionCount by N per cycle (a 6-variant + // story would skip DEVELOPING straight to SUSTAINED, since the read + // path treats mentionCount as +1/cycle) and let whichever member + // iterated last overwrite the representative fields nondeterministically. + // Representative = highest importanceScore, tie-break newest publishedAt + // then title — deterministic for a given batch. Per-MEMBER writes that + // are set-shaped stay per item: SADD source (distinct-source set is the + // point of corroboration) and ZADD peak GT (max is idempotent). + const representativeByHash = new Map(); + for (let i = 0; i < items.length; i++) { + const hash = hashes[i]!; + const item = items[i]!; + const current = representativeByHash.get(hash); + if ( + !current + || item.importanceScore > current.importanceScore + || (item.importanceScore === current.importanceScore && item.publishedAt > current.publishedAt) + || (item.importanceScore === current.importanceScore && item.publishedAt === current.publishedAt + && item.title < current.title) + ) { + representativeByHash.set(hash, item); + } + } + + const writtenHashes = new Set(); for (let batchStart = 0; batchStart < items.length; batchStart += STORY_BATCH_SIZE) { const batch = items.slice(batchStart, batchStart + STORY_BATCH_SIZE); const commands: Array> = []; @@ -1128,18 +1158,35 @@ async function writeStoryTracking(items: ParsedItem[], variant: string, lang: st const nowStr = String(now); const ttl = STORY_TTL; - const hsetFields = buildStoryTrackHsetFields(item, nowStr, score); + if (!writtenHashes.has(hash)) { + writtenHashes.add(hash); + const representative = representativeByHash.get(hash) ?? item; + const hsetFields = buildStoryTrackHsetFields(representative, nowStr, representative.importanceScore); + commands.push( + ['HINCRBY', trackKey, 'mentionCount', '1'], + ['HSET', trackKey, ...hsetFields], + ['HSETNX', trackKey, 'firstSeen', nowStr], + ['EXPIRE', trackKey, ttl], + ['ZADD', accKey, nowStr, hash], + ); + // #4924: alias rows for every member exact-title hash -> the FINAL + // (post-adoption) canonical, story-track TTL — next cycle's + // adoption source. Includes the canonical's own hash. + for (const memberHash of memberHashesByFinal?.get(hash) ?? []) { + commands.push(['SET', STORY_ALIAS_KEY(memberHash), hash, 'EX', ttl]); + } + } commands.push( - ['HINCRBY', trackKey, 'mentionCount', '1'], - ['HSET', trackKey, ...hsetFields], - ['HSETNX', trackKey, 'firstSeen', nowStr], ['ZADD', peakKey, 'GT', score, 'peak'], ['SADD', sourcesKey, item.source], - ['EXPIRE', trackKey, ttl], + // #4924 review P2 (TTL ordering): EXPIRE must follow the SADD/ZADD + // that CREATE these keys — EXPIRE on a missing key is a no-op, so + // the pre-block ordering left brand-new story:sources/story:peak + // keys persistent forever. Idempotent per member; kept adjacent to + // the creating writes so no future reorder can reopen the leak. ['EXPIRE', sourcesKey, ttl], ['EXPIRE', peakKey, ttl], - ['ZADD', accKey, nowStr, hash], ); } @@ -1243,19 +1290,64 @@ async function buildDigest(variant: string, lang: string): Promise>(); + // #4919: fuzzy story identity. Items are clustered by the shared + // story-identity similarity (edit-tolerant: suffixes, truncations, + // qualifier swaps, reorders, morphology) and every cluster member + // shares one canonical titleHash + a cluster-wide corroboration + // count. The previous exact sha256(normalizeTitle) identity forked a + // story on ANY wording edit, so corroboration only counted verbatim + // wire syndication — deflating importanceScore's corroboration + // signal and the BREAKING/DEVELOPING phase tracker. Singleton + // clusters hash exactly as before, so story:track keys for + // uncorroborated stories are unchanged. + const identityByItem = await assignStoryIdentity(allItems, normalizeTitle, sha256Hex); + + // #4924 review P1: adopt a LIVE canonical before assigning hashes. + // Alias rows (memberHash -> canonicalHash, story-track TTL) written by + // previous cycles let a cluster keep its story identity when the + // member that anchored the canonical drops out of the batch. One + // batched read for all member hashes; failures degrade to + // batch-derived canonicals (pre-adoption behavior). + const allMemberHashes = new Set(); + for (const identity of identityByItem.values()) { + for (const h of identity.memberTitleHashes ?? []) allMemberHashes.add(h); + } + const aliasTargetByHash = new Map(); + if (allMemberHashes.size > 0) { + const aliasHashes = [...allMemberHashes]; + const aliasResults = await runRedisPipeline(aliasHashes.map((h) => ['GET', STORY_ALIAS_KEY(h)])); + for (let i = 0; i < aliasHashes.length; i++) { + const target = aliasResults[i]?.result; + if (typeof target === 'string' && target.length > 0) aliasTargetByHash.set(aliasHashes[i]!, target); + } + } + await Promise.all(allItems.map(async (item) => { - const hash = await sha256Hex(normalizeTitle(item.title)); - item.titleHash = hash; - const sources = corroborationMap.get(hash) ?? new Set(); - sources.add(item.source); - corroborationMap.set(hash, sources); + const identity = identityByItem.get(item); + if (identity) { + item.titleHash = adoptExistingCanonical(identity.memberTitleHashes, identity.titleHash, aliasTargetByHash); + item.corroborationCount = identity.corroborationCount; + } else { + // Defensive: assignStoryIdentity covers every input by + // construction; degrade to the pre-#4919 exact identity if not — + // and say so, or a future coverage-invariant break is invisible. + console.warn( + `[digest] story-identity coverage miss — exact-hash fallback for "${item.title.slice(0, 60)}"`, + ); + item.titleHash = await sha256Hex(normalizeTitle(item.title)); + item.corroborationCount = 1; + } })); + // Final(post-adoption) hash -> member exact-title hashes, consumed by + // writeStoryTracking to persist next cycle's alias rows. + const memberHashesByFinal = new Map>(); for (const item of allItems) { - item.corroborationCount = corroborationMap.get(item.titleHash!)?.size ?? 1; + const identity = identityByItem.get(item); + if (!identity || !item.titleHash) continue; + let set = memberHashesByFinal.get(item.titleHash); + if (!set) { set = new Set(); memberHashesByFinal.set(item.titleHash, set); } + for (const h of identity.memberTitleHashes ?? []) set.add(h); } // Enrich ALL items with the AI classification cache BEFORE scoring so that @@ -1334,7 +1426,7 @@ async function buildDigest(variant: string, lang: string): Promise new Map()); // Write story tracking. Errors never fail the digest build. - await writeStoryTracking(allSliced, variant, lang, titleHashes).catch((err: unknown) => + await writeStoryTracking(allSliced, variant, lang, titleHashes, memberHashesByFinal).catch((err: unknown) => console.warn('[digest] story tracking write failed:', err), ); @@ -1342,7 +1434,8 @@ async function buildDigest(variant: string, lang: string): Promise { const hash = item.titleHash!; - const sourceCount = corroborationMap.get(hash)?.size ?? 1; + // #4919: cluster-wide source count assigned by assignStoryIdentity. + const sourceCount = item.corroborationCount ?? 1; const stale = storyTracks.get(hash); // Merge stale state + this cycle's HINCRBY to get the current mentionCount. // New stories (stale = undefined) start at mentionCount=1 this cycle. diff --git a/shared/story-identity.d.ts b/shared/story-identity.d.ts new file mode 100644 index 0000000000..3e2e60ab68 --- /dev/null +++ b/shared/story-identity.d.ts @@ -0,0 +1,30 @@ +/** + * Types for shared/story-identity.js — the single "same news story?" + * similarity definition (#4919). See the .js module doc for method + * details and tuning provenance. + */ + +/** Dual-view story vector — opaque; produce with storyVector(), compare + * with cosineSimilarity(). `u` = uniform-weight view, `b` = + * entity/number-boosted view; similarity is the min of both cosines. */ +export interface StoryVector { + u: Float64Array; + b: Float64Array; + /** Content-token set (lexical vectors only) — drives the containment + * rescue for truncated headlines; absent on semantic-provider vectors. */ + t?: Set; +} + +export const STORY_SIMILARITY_THRESHOLD: number; + +export function normalizeStoryText(text: string): string; +export function stripAttributionSuffix(text: string): string; +export function candidateTokens(text: string): Set; +export function setStoryVectorProvider( + provider: ((text: string) => StoryVector | null) | null, +): void; +export function storyVector(text: string): StoryVector | null; +export function cosineSimilarity(a: StoryVector | null, b: StoryVector | null): number; +export function storySimilarity(textA: string, textB: string): number; +export function isSameStory(textA: string, textB: string, threshold?: number): boolean; +export function clusterTexts(texts: string[], opts?: { threshold?: number }): number[][]; diff --git a/shared/story-identity.js b/shared/story-identity.js new file mode 100644 index 0000000000..b77769e0f2 --- /dev/null +++ b/shared/story-identity.js @@ -0,0 +1,443 @@ +/** + * story-identity — the ONE similarity definition for "are these the same + * news story?" (#4919, Bet 1 of the 2026-07-05 strategic review). + * + * Before this module the codebase answered that question three different + * ways: scripts/_clustering.mjs (title-token Jaccard ≥ 0.5), + * server/worldmonitor/news/v1/dedup.mjs (word-overlap > 0.6 of the smaller + * set), and list-feed-digest.ts story tracking (EXACT sha256 of the + * normalized title — so any wording edit forked the story and deflated + * corroboration). All three now delegate here. + * + * ── Method ────────────────────────────────────────────────────────────── + * DUAL-VIEW feature-hashed lexical vectors; similarity = min of the two + * views' cosines (see lexicalStoryVector for why two views). Features: + * - word tokens (weight 2.0) — core lexical identity + * - word bigrams (weight 1.5) — order/direction ("ukraine + * drone" vs "russian drone" separates actor-flipped headlines that + * bag-of-words alone cannot) + * - char 4-grams per token (weight 1.0) — morphology fuzz + * (iran/iranian, sanction/sanctions) + * - char bigrams for non-ASCII tokens — CJK and other unsegmented + * scripts get no whitespace tokens, so bigrams carry the signal + * hashed (signed FNV-1a) into 512 dims and L2-normalized. Deterministic, + * dependency-free, script-agnostic, ~µs per title. + * + * This is an EDIT-TOLERANT identity, not a semantic one: it merges the + * real-world corroboration killers (source suffixes, truncations, + * qualifier edits, reorders, morphology) and keeps distinct events apart. + * It can NOT merge a full cross-language paraphrase ("Iran threatens…" / + * "Teherán amenaza…") — that requires a semantic embedding provider, + * which plugs in behind `setStoryVectorProvider()` without touching any + * consumer. Known hard limit either way: two events differing by one + * token ("12th sanctions package" vs "13th sanctions package") are not + * separable by similarity alone; the 96h ingest window bounds the damage. + * + * Mirrored byte-for-byte to scripts/shared/story-identity.js (enforced by + * tests/scripts-shared-mirror.test.mjs — Railway seed bundles deploy with + * rootDirectory=scripts and cannot see repo-root shared/). + */ + +const DIM = 512; + +// Tuned on the labeled pair set in tests/story-identity.test.mjs +// (edit-variant positives vs same-topic-different-event negatives). The +// test asserts full separation with margin on both sides; retune there +// if the vectorizer changes. +export const STORY_SIMILARITY_THRESHOLD = 0.615; + +const WEIGHT_TOKEN = 2.0; +const WEIGHT_BIGRAM = 1.5; +const WEIGHT_CHARGRAM = 1.0; +// Discriminative boosts, applied to the token feature only (bigrams keep +// their flat weight — they already encode order). Without these, a +// one-entity swap ("Turkey hikes rates to 50%" vs "Argentina hikes rates +// to 50%") scores ~0.82 because the shared verb/number mass dominates; +// capitalized-in-raw-text tokens are entity-shaped and numbers are +// event parameters (magnitudes, percentages, ordinals), so both carry +// the discriminating signal. In Title Case or ALL-CAPS headlines every +// token gets the boost — uniform scaling, which cosine ignores — so the +// heuristic only sharpens sentence-case headlines and never hurts. +const BOOST_ENTITY = 3.0; +const BOOST_NUMBER = 2.0; + +/** FNV-1a 32-bit over a string, with a seed so we can derive two + * independent hashes (index + sign) from one feature. */ +function fnv1a(str, seed) { + let h = (0x811c9dc5 ^ seed) >>> 0; + for (let i = 0; i < str.length; i++) { + h ^= str.charCodeAt(i); + h = Math.imul(h, 0x01000193) >>> 0; + } + return h >>> 0; +} + +/** + * Generic story-text normalization: lowercase, strip everything that is + * not a Unicode letter/number, collapse whitespace. Callers that know + * about source-attribution suffixes ("… - Reuters") strip those BEFORE + * calling (list-feed-digest's normalizeTitle already does). + * @param {string} text + * @returns {string} + */ +export function normalizeStoryText(text) { + return (text || '') + .toLowerCase() + .replace(/[^\p{L}\p{N}\s]/gu, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +/** @param {string} token */ +function isNonAscii(token) { + for (let i = 0; i < token.length; i++) { + if (token.charCodeAt(i) > 127) return true; + } + return false; +} + +/** + * Tokens used for inverted-index candidate generation by clustering + * callers (cheap pre-filter: only pairs sharing ≥1 token are scored). + * ASCII tokens shorter than 3 chars are dropped (stopword-weight noise); + * non-ASCII tokens are kept whole AND as char bigrams so unsegmented + * scripts still produce index keys. + * @param {string} text + * @returns {Set} + */ +export function candidateTokens(text) { + const out = new Set(); + const clamped = stripAttributionSuffix(text).slice(0, MAX_IDENTITY_CHARS); + for (const tok of normalizeStoryText(clamped).split(' ')) { + if (!tok) continue; + if (isNonAscii(tok)) { + out.add(tok); + for (let i = 0; i + 2 <= tok.length; i++) out.add(tok.slice(i, i + 2)); + } else if (tok.length >= 3) { + out.add(tok); + } + } + return out; +} + +// Trailing source-attribution suffixes ("… - Reuters", "… - example.com") +// must not enter the vector: Google-News wrapper titles carry them on +// EVERY item, so the publisher token (capitalized → entity-boosted ×3) +// adds shared mass across DISTINCT same-publisher stories and pulls them +// toward a false merge (cross-model review finding, PR #4924). Mirrors +// list-feed-digest's normalizeTitle suffix rules, but case-preserving. +const ATTRIBUTION_SUFFIX_RES = [ + /\s*[-\u2013\u2014|]\s*[\w\s.]+\.(?:com|org|net|co\.uk)\s*$/i, + /\s*[-\u2013\u2014|]\s*(?:reuters|ap news|bbc|cnn|al jazeera|france 24|dw news|pbs newshour|cbs news|nbc|abc|associated press|the guardian|nos nieuws|tagesschau|cnbc|the national)\s*$/i, +]; + +// Unbounded feed titles feed char-4gram loops inside a 25s serverless +// budget; clamp AFTER suffix stripping. 300 chars ≈ 3× a long headline. +const MAX_IDENTITY_CHARS = 300; + +/** @param {string} text @returns {string} */ +export function stripAttributionSuffix(text) { + let out = text || ''; + for (const re of ATTRIBUTION_SUFFIX_RES) out = out.replace(re, ''); + return out; +} + +/** + * Content tokens WITH the discriminative flags read from the raw + * (pre-lowercase) text. Callers should pass raw titles — lowercasing + * upstream destroys the entity signal (harmless, but the boost is lost). + * @param {string} text + * @returns {Array<{ tok: string; boost: number }>} + */ +function contentTokens(text) { + const kept = []; + const clamped = stripAttributionSuffix(text).slice(0, MAX_IDENTITY_CHARS); + for (const raw of clamped.split(/\s+/)) { + // Strip everything that is not a Unicode letter/number, keeping the + // original case so the entity heuristic can read it. + const clean = raw.replace(/[^\p{L}\p{N}]/gu, ''); + if (!clean) continue; + const tok = clean.toLowerCase(); + if (!isNonAscii(tok) && tok.length < 3) continue; + const capitalized = /^\p{Lu}/u.test(clean); + const hasDigit = /\p{N}/u.test(clean); + const boost = hasDigit ? BOOST_NUMBER : capitalized ? BOOST_ENTITY : 1; + kept.push({ tok, boost }); + } + return kept; +} + +/** @param {Float64Array} vec @param {string} feature @param {number} weight */ +function addFeature(vec, feature, weight) { + const idx = fnv1a(feature, 0) % DIM; + const sign = (fnv1a(feature, 0x9e3779b9) & 1) === 1 ? 1 : -1; + vec[idx] += sign * weight; +} + +/** @param {Float64Array} vec */ +function l2normalize(vec) { + let norm = 0; + for (let i = 0; i < DIM; i++) norm += vec[i] * vec[i]; + norm = Math.sqrt(norm); + if (norm === 0) return null; + for (let i = 0; i < DIM; i++) vec[i] /= norm; + return vec; +} + +/** + * The default lexical vectorizer — DUAL VIEW. Returns two L2-normalized + * 512-dim views of the same text: + * - `u` (uniform): every token feature at flat weight. Sensitive to + * action/verb substitutions ("seizes tanker" vs "threatens to + * close") that entity weighting would wash out. + * - `b` (boosted): entity-shaped (capitalized-in-raw) tokens ×3 and + * numeric tokens ×2. Sensitive to one-entity swaps ("Turkey hikes + * rates…" vs "Argentina hikes rates…") that flat weighting scores + * ~0.82 because the shared verb mass dominates. + * A pair is the same story only when BOTH views agree (similarity = + * min of the two cosines) — each view catches the failure mode the + * other is blind to. Tuned on the labeled pair set in + * tests/story-identity.test.mjs: min positive 0.634, max negative + * 0.595 with THRESHOLD 0.615 between them. + * + * Returns null for texts with no usable tokens (callers treat null as + * "cannot match" — never same-story). + * @param {string} text + * @returns {{ u: Float64Array; b: Float64Array } | null} + */ +function lexicalStoryVector(text) { + const tokens = contentTokens(text); + if (tokens.length === 0) return null; + const u = new Float64Array(DIM); + const b = new Float64Array(DIM); + for (let i = 0; i < tokens.length; i++) { + const { tok, boost } = tokens[i]; + addFeature(u, `w:${tok}`, WEIGHT_TOKEN); + addFeature(b, `w:${tok}`, WEIGHT_TOKEN * boost); + if (i + 1 < tokens.length) { + const bigram = `b:${tok} ${tokens[i + 1].tok}`; + addFeature(u, bigram, WEIGHT_BIGRAM); + addFeature(b, bigram, WEIGHT_BIGRAM); + } + if (isNonAscii(tok)) { + // Unsegmented-script fallback: char bigrams of the raw token. + for (let j = 0; j + 2 <= tok.length; j++) { + const g = `c2:${tok.slice(j, j + 2)}`; + addFeature(u, g, WEIGHT_CHARGRAM); + addFeature(b, g, WEIGHT_CHARGRAM); + } + } + if (tok.length >= 4) { + const padded = `<${tok}>`; + for (let j = 0; j + 4 <= padded.length; j++) { + const g = `c4:${padded.slice(j, j + 4)}`; + addFeature(u, g, WEIGHT_CHARGRAM); + addFeature(b, g, WEIGHT_CHARGRAM); + } + } + } + const un = l2normalize(u); + const bn = l2normalize(b); + if (!un || !bn) return null; + // Token set rides along for the containment rescue in + // cosineSimilarity — severe RSS truncation (a headline cut to ~40% of + // its tokens) drops the cosine below threshold even though the short + // form is a strict subset of the long form. The old word-overlap + // dedup metric (|∩|/min) handled exactly this class; keep that + // guarantee via token containment. + return { u: un, b: bn, t: new Set(tokens.map((entry) => entry.tok)) }; +} + +/** Active vectorizer — swappable for a semantic embedding provider. */ +let activeVectorizer = lexicalStoryVector; + +/** + * Plug in a semantic embedding provider (must be synchronous or the + * caller precomputes; must return `{ u, b }` of L2-normalized + * Float64Arrays of a consistent dimension — a single-embedding provider + * sets u === b — or null). Pass null to restore the default lexical + * vectorizer. Consumers never change — only the vector source. + * @param {((text: string) => { u: Float64Array; b: Float64Array } | null) | null} provider + */ +export function setStoryVectorProvider(provider) { + activeVectorizer = typeof provider === 'function' ? provider : lexicalStoryVector; +} + +/** + * @param {string} text + * @returns {{ u: Float64Array; b: Float64Array } | null} dual-view story + * vector (opaque — pass to cosineSimilarity), or null when the text + * has no usable signal. + */ +export function storyVector(text) { + return activeVectorizer(text); +} + +/** @param {Float64Array} a @param {Float64Array} b */ +function dot(a, b) { + if (a.length !== b.length) return 0; + let d = 0; + for (let i = 0; i < a.length; i++) d += a[i] * b[i]; + return d; +} + +/** + * Similarity of two dual-view story vectors: the MIN of the uniform-view + * and boosted-view cosines — a pair is the same story only when both + * views agree. Null vectors never match anything. + * @param {{ u: Float64Array; b: Float64Array } | null} a + * @param {{ u: Float64Array; b: Float64Array } | null} b + * @returns {number} + */ +// Containment rescue floor: a title whose content tokens are ≥90% +// contained in the other's (with at least 4 tokens on the smaller side, +// so fragments like "Iran" can't rescue) IS the same story — the +// truncated-wire-copy class the old |∩|/min dedup metric guaranteed. +const CONTAINMENT_RESCUE_MIN_TOKENS = 4; +const CONTAINMENT_RESCUE_RATIO = 0.9; +const CONTAINMENT_RESCUE_SCORE = 0.9; + +export function cosineSimilarity(a, b) { + if (!a || !b) return 0; + const score = Math.min(dot(a.u, b.u), dot(a.b, b.b)); + // Rescue only applies to lexical vectors carrying token sets — a + // semantic provider's vectors skip it (semantic cosine already + // handles truncation). + if (score < CONTAINMENT_RESCUE_SCORE && a.t && b.t) { + const [small, large] = a.t.size <= b.t.size ? [a.t, b.t] : [b.t, a.t]; + if (small.size >= CONTAINMENT_RESCUE_MIN_TOKENS) { + let shared = 0; + for (const tok of small) { + if (large.has(tok)) shared++; + } + if (shared / small.size >= CONTAINMENT_RESCUE_RATIO) { + return CONTAINMENT_RESCUE_SCORE; + } + } + } + return score; +} + +/** + * Convenience: similarity of two raw texts. + * @param {string} textA @param {string} textB + * @returns {number} + */ +export function storySimilarity(textA, textB) { + return cosineSimilarity(storyVector(textA), storyVector(textB)); +} + +/** + * @param {string} textA @param {string} textB + * @param {number} [threshold] + * @returns {boolean} + */ +export function isSameStory(textA, textB, threshold = STORY_SIMILARITY_THRESHOLD) { + return storySimilarity(textA, textB) >= threshold; +} + +// A token shared by more than this many titles carries no clustering +// signal (it is the batch's "the") but drives O(bucket²) pair scoring — +// an adversarial or organic hot-entity spike (thousands of titles naming +// one entity) would otherwise burn seconds of CPU inside the digest +// handler's 25s budget. Pairs joined ONLY by ultra-hot tokens almost +// always share a rarer token too. +const MAX_CANDIDATE_BUCKET = 250; + +/** + * Cluster texts into same-story groups: connected components over the + * "similarity ≥ threshold" edge set (union-find), with inverted-index + * candidate generation so only pairs sharing ≥1 token are scored. + * + * Connected components — NOT the greedy first-seed pass the legacy + * _clustering.mjs used — because component membership is independent of + * input order: feed arrival order varies run to run, and under greedy + * assignment a chain (A~B, B~C, A≁C) could land C in or out of A's + * cluster depending on which seeded first, churning the canonical + * story:track identity downstream (cross-model review finding, + * PR #4924). Transitive chains merge by design; the threshold's + * precision bounds chain length in practice. + * @param {string[]} texts + * @param {{ threshold?: number }} [opts] + * @returns {number[][]} clusters of indices into `texts`, ordered by + * smallest member index; members ascending + */ +export function clusterTexts(texts, opts = {}) { + const threshold = typeof opts.threshold === 'number' ? opts.threshold : STORY_SIMILARITY_THRESHOLD; + const vectors = texts.map((t) => storyVector(t)); + const tokenSets = texts.map((t) => candidateTokens(t)); + + // Exact-duplicate pre-union (#4924 external review): identical + // normalized texts union unconditionally BEFORE the candidate scan. + // Without this, a mega-story (e.g. 251 verbatim syndications) makes + // every shared token bucket exceed MAX_CANDIDATE_BUCKET, no pairs get + // scored, and the most-corroborated story of the day degrades to + // singletons — the exact case corroboration exists for. + const byExactText = new Map(); + + const invertedIndex = new Map(); + for (let i = 0; i < texts.length; i++) { + for (const token of tokenSets[i]) { + const bucket = invertedIndex.get(token); + if (bucket) bucket.push(i); + else invertedIndex.set(token, [i]); + } + } + + const parent = new Array(texts.length); + for (let i = 0; i < texts.length; i++) parent[i] = i; + + const find = (x) => { + let root = x; + while (parent[root] !== root) root = parent[root]; + while (parent[x] !== root) { + const next = parent[x]; + parent[x] = root; + x = next; + } + return root; + }; + const union = (a, b) => { + const ra = find(a); + const rb = find(b); + if (ra === rb) return; + // Deterministic: smaller index becomes the root. + if (ra < rb) parent[rb] = ra; + else parent[ra] = rb; + }; + + for (let i = 0; i < texts.length; i++) { + const normalized = normalizeStoryText(texts[i]); + if (!normalized) continue; + const first = byExactText.get(normalized); + if (first === undefined) byExactText.set(normalized, i); + else union(first, i); + } + + for (let i = 0; i < texts.length; i++) { + if (!vectors[i]) continue; + const candidates = new Set(); + for (const token of tokenSets[i]) { + const bucket = invertedIndex.get(token); + if (!bucket || bucket.length > MAX_CANDIDATE_BUCKET) continue; + for (const idx of bucket) { + if (idx > i) candidates.add(idx); + } + } + for (const j of candidates) { + if (find(i) === find(j)) continue; + if (cosineSimilarity(vectors[i], vectors[j]) >= threshold) union(i, j); + } + } + + const byRoot = new Map(); + for (let i = 0; i < texts.length; i++) { + const root = find(i); + const members = byRoot.get(root); + if (members) members.push(i); + else byRoot.set(root, [i]); + } + return Array.from(byRoot.entries()) + .sort((a, b) => a[0] - b[0]) + .map(([, members]) => members); +} diff --git a/tests/scripts-shared-mirror.test.mjs b/tests/scripts-shared-mirror.test.mjs index 3aae0f0dd3..0ec032d1be 100644 --- a/tests/scripts-shared-mirror.test.mjs +++ b/tests/scripts-shared-mirror.test.mjs @@ -21,6 +21,7 @@ const MIRRORED_FILES = [ 'geography.js', 'iso2-to-region.json', 'iso3-to-iso2.json', + 'story-identity.js', 'un-to-iso2.json', ]; @@ -53,6 +54,7 @@ describe('regional snapshot seed scripts use scripts/shared/ (not repo-root shar // shared/) will ERR_MODULE_NOT_FOUND at runtime on Railway because the // shared/ dir is not copied into the deploy root. const FILES_THAT_MUST_USE_MIRROR = [ + 'scripts/_clustering.mjs', 'scripts/seed-regional-snapshots.mjs', 'scripts/regional-snapshot/actor-scoring.mjs', 'scripts/regional-snapshot/balance-vector.mjs', diff --git a/tests/server-handlers.test.mjs b/tests/server-handlers.test.mjs index c87c7b233a..4df556147c 100644 --- a/tests/server-handlers.test.mjs +++ b/tests/server-handlers.test.mjs @@ -131,9 +131,10 @@ describe('headline deduplication', () => { 'Russia launches missile strike on Ukrainian energy infrastructure overnight', 'EU approves new sanctions package against Russia', ]; - // Words >= 4 chars for headline 1: russia, launches, missile, strike, ukrainian, energy, infrastructure, targets (8) - // Words >= 4 chars for headline 2: russia, launches, missile, strike, ukrainian, energy, infrastructure, overnight (8) - // Intersection: 7/8 = 0.875 > 0.6 threshold + // #4919: similarity now comes from shared/story-identity.js (dual-view + // cosine >= STORY_SIMILARITY_THRESHOLD) — a one-token tail swap on an + // otherwise identical headline is squarely inside the edit-variant + // class it must merge. const result = deduplicateHeadlines(headlines); assert.equal(result.length, 2, 'Should deduplicate near-identical headlines'); assert.equal(result[0], headlines[0], 'Should keep the first occurrence'); diff --git a/tests/story-identity.test.mjs b/tests/story-identity.test.mjs new file mode 100644 index 0000000000..0b3328983c --- /dev/null +++ b/tests/story-identity.test.mjs @@ -0,0 +1,461 @@ +// shared/story-identity.js — the single "same news story?" definition +// (#4919). The labeled pair set below is the tuning ground truth for +// STORY_SIMILARITY_THRESHOLD: if you change the vectorizer, weights, or +// threshold, this suite tells you whether separation still holds. + +import assert from 'node:assert/strict'; +import { describe, it, afterEach } from 'node:test'; +import { + STORY_SIMILARITY_THRESHOLD, + normalizeStoryText, + stripAttributionSuffix, + candidateTokens, + storyVector, + cosineSimilarity, + storySimilarity, + isSameStory, + clusterTexts, + setStoryVectorProvider, +} from '../shared/story-identity.js'; + +afterEach(() => setStoryVectorProvider(null)); + +// ── Labeled pairs (tuning ground truth) ──────────────────────────────────── +// +// POSITIVES: the edit-variant class this identity MUST merge — source +// suffixes, truncations, qualifier swaps, reorders, light morphology. +// These are the real-world corroboration killers under exact-hash identity. +const POSITIVE_PAIRS = [ + ['Fed holds interest rates steady amid inflation concerns', 'Fed holds rates steady as inflation concerns persist'], + ['Magnitude 6.8 earthquake strikes northern Chile', '6.8-magnitude earthquake hits northern Chile'], + ['EU approves 12th sanctions package against Russia', 'European Union approves 12th sanctions package on Russia'], + ['Ukraine drone strike hits Russian oil refinery in Ryazan region', 'Ukraine drone strike hits Russian oil refinery'], + ['Iran threatens to close Strait of Hormuz if US blockade continues', 'Iran threatens to close Strait of Hormuz — live updates'], + ['Apple unveils new AI features at WWDC keynote', 'At WWDC keynote, Apple unveils new AI features'], + ['Iranian officials threaten Hormuz closure over sanctions', 'Iran officials threaten Hormuz closure over sanctions'], + ['Nigeria fuel subsidy protests spread to Lagos as unions join', 'Nigeria fuel subsidy protests spread to Lagos'], + ['Turkey hikes interest rates to 50% in surprise move', 'Turkey hikes rates to 50% in surprise move'], + ['China exports fall 7.5% in June, worse than expected', 'Chinese exports fell 7.5% in June, worse than expected'], + // Severe RSS truncation (>=50% token drop) — the containment-rescue + // class the old word-overlap dedup metric guaranteed (#4924 review). + ['Nigeria fuel subsidy protests spread to Lagos as unions join nationwide strike over cost of living', 'Nigeria fuel subsidy protests spread to Lagos'], + ['Turkey central bank hikes interest rates to 50% in surprise move to combat runaway inflation pressures', 'Turkey central bank hikes interest rates'], + // Source-attribution suffix must not shift identity (#4924 cross-model). + ['Iran threatens to close Strait of Hormuz - Reuters', 'Iran threatens to close Strait of Hormuz'], +]; + +// NEGATIVES: same-topic-DIFFERENT-event pairs that must stay apart — +// entity swaps, action swaps, parameter swaps, actor-direction flips. +const NEGATIVE_PAIRS = [ + ['Iran seizes oil tanker in Strait of Hormuz', 'Iran threatens to close Strait of Hormuz'], + ['Fed holds rates steady amid inflation concerns', 'Fed cuts rates by 25 basis points amid slowing economy'], + ['Magnitude 6.8 earthquake strikes northern Chile', 'Magnitude 5.9 earthquake strikes southern Peru'], + ['Ukraine drone strike hits Russian oil refinery', 'Russian drone strike hits Ukrainian energy grid'], + ['Apple unveils new AI features at WWDC keynote', 'Google unveils new AI features at I/O keynote'], + ['Turkey hikes interest rates to 50% in surprise move', 'Argentina hikes interest rates to 50% in surprise move'], + ['Nigeria fuel subsidy protests spread to Lagos', 'Kenya tax protests spread to Nairobi'], + ['US imposes new sanctions on Iranian oil exports', 'US lifts sanctions on Venezuelan oil exports'], + ['Israel strikes Hezbollah targets in southern Lebanon', 'Hezbollah strikes Israeli positions in northern Israel'], + // Shared publisher suffix must NOT pull distinct stories together — + // pre-fix, the entity-boosted "Reuters" token added shared mass to + // every same-wrapper pair (#4924 cross-model finding). + ['Apple unveils new AI features at WWDC keynote - Reuters', 'Google unveils new AI features at I/O keynote - Reuters'], +]; + +// KNOWN LIMIT (documented, deliberately NOT asserted as separable): two +// events differing by ONE unboosted content token sit above the +// threshold — "China exports fall 7.5%" vs "China imports fall 7.5%", +// "12th sanctions package" vs "13th". No lexical similarity can order +// these below genuine rewrites of one story; the 96h ingest window and +// entity-corroboration signals bound the damage. Revisit when a semantic +// provider lands behind setStoryVectorProvider. + +describe('labeled-pair separation (tuning ground truth)', () => { + it('every edit-variant positive pair clears the threshold', () => { + for (const [a, b] of POSITIVE_PAIRS) { + const sim = storySimilarity(a, b); + assert.ok( + sim >= STORY_SIMILARITY_THRESHOLD, + `expected same-story (${sim.toFixed(3)} >= ${STORY_SIMILARITY_THRESHOLD}): "${a}" ~ "${b}"`, + ); + } + }); + + it('every distinct-event negative pair stays below the threshold', () => { + for (const [a, b] of NEGATIVE_PAIRS) { + const sim = storySimilarity(a, b); + assert.ok( + sim < STORY_SIMILARITY_THRESHOLD, + `expected distinct (${sim.toFixed(3)} < ${STORY_SIMILARITY_THRESHOLD}): "${a}" vs "${b}"`, + ); + } + }); + + it('separation holds with margin on both sides (retune trip-wire)', () => { + const minPos = Math.min(...POSITIVE_PAIRS.map(([a, b]) => storySimilarity(a, b))); + const maxNeg = Math.max(...NEGATIVE_PAIRS.map(([a, b]) => storySimilarity(a, b))); + assert.ok(minPos - STORY_SIMILARITY_THRESHOLD >= 0.015, `positive floor too close: ${minPos.toFixed(3)}`); + assert.ok(STORY_SIMILARITY_THRESHOLD - maxNeg >= 0.015, `negative ceiling too close: ${maxNeg.toFixed(3)}`); + }); +}); + +describe('storyVector / cosineSimilarity', () => { + it('identical texts are similarity 1', () => { + const sim = storySimilarity('Iran threatens to close Strait of Hormuz', 'Iran threatens to close Strait of Hormuz'); + assert.ok(Math.abs(sim - 1) < 1e-9); + }); + + it('empty/garbage text yields null vector and zero similarity', () => { + assert.equal(storyVector(''), null); + assert.equal(storyVector(' —— !!'), null); + assert.equal(storySimilarity('', 'Iran threatens Hormuz'), 0); + assert.equal(cosineSimilarity(null, storyVector('Iran threatens Hormuz')), 0); + }); + + it('is symmetric', () => { + const a = 'Fed holds rates steady amid inflation concerns'; + const b = 'Fed holds interest rates steady'; + assert.ok(Math.abs(storySimilarity(a, b) - storySimilarity(b, a)) < 1e-12); + }); + + it('unsegmented scripts (CJK) still produce vectors and match near-duplicates', () => { + const a = '日本銀行が金利を引き上げ、市場に衝撃'; + const b = '日本銀行が金利を引き上げ'; + assert.ok(storyVector(a), 'CJK title must vectorize'); + assert.ok(storySimilarity(a, b) > storySimilarity(a, '米国大統領がメキシコ国境を視察')); + }); + + it('case-only differences do not change identity (boost is view-internal)', () => { + const sim = storySimilarity( + 'IRAN THREATENS TO CLOSE STRAIT OF HORMUZ', + 'Iran threatens to close Strait of Hormuz', + ); + assert.ok(sim >= STORY_SIMILARITY_THRESHOLD, `all-caps variant must merge (got ${sim.toFixed(3)})`); + }); +}); + +describe('clusterTexts', () => { + it('groups edit variants and keeps distinct events apart', () => { + const texts = [ + 'Iran threatens to close Strait of Hormuz if US blockade continues', + 'Iran threatens to close Strait of Hormuz — live updates', + 'Stock market rallies on tech earnings report', + 'Iran seizes oil tanker in Strait of Hormuz', + ]; + const clusters = clusterTexts(texts); + const byMember = new Map(); + clusters.forEach((cluster, ci) => cluster.forEach((i) => byMember.set(i, ci))); + assert.equal(byMember.get(0), byMember.get(1), 'variants must share a cluster'); + assert.notEqual(byMember.get(0), byMember.get(2), 'unrelated story must not join'); + assert.notEqual(byMember.get(0), byMember.get(3), 'same-topic different event must not join'); + assert.equal(clusters.flat().length, texts.length, 'every index appears exactly once'); + }); + + it('is deterministic and order-stable for a fixed input', () => { + const texts = [ + 'Turkey hikes interest rates to 50% in surprise move', + 'Turkey hikes rates to 50% in surprise move', + 'Kenya tax protests spread to Nairobi', + ]; + assert.deepEqual(clusterTexts(texts), clusterTexts(texts)); + }); + + it('respects an explicit threshold override', () => { + const texts = ['Fed holds interest rates steady', 'Fed holds rates steady']; + assert.equal(clusterTexts(texts, { threshold: 0.999 }).length, 2); + assert.equal(clusterTexts(texts).length, 1); + }); +}); + +describe('candidateTokens / normalizeStoryText', () => { + it('drops short ASCII tokens and keeps non-ASCII with bigrams', () => { + const toks = candidateTokens('US to cut rates 日本'); + assert.ok(!toks.has('to')); + assert.ok(toks.has('rates')); + assert.ok(toks.has('日本')); + assert.ok(toks.has('日本'.slice(0, 2))); + }); + + it('normalizeStoryText strips punctuation and collapses whitespace', () => { + assert.equal(normalizeStoryText(' Fed — holds, rates! '), 'fed holds rates'); + }); +}); + +describe('setStoryVectorProvider (semantic upgrade seam)', () => { + it('routes storyVector through the provider and restores on null', () => { + const fixed = { u: new Float64Array(4).fill(0.5), b: new Float64Array(4).fill(0.5) }; + setStoryVectorProvider(() => fixed); + assert.equal(storyVector('anything'), fixed); + assert.ok(Math.abs(storySimilarity('a b c', 'x y z') - 1) < 1e-9, 'provider vectors drive similarity'); + setStoryVectorProvider(null); + assert.notEqual(storyVector('Iran threatens Hormuz closure'), fixed); + }); +}); + +// ── assignStoryIdentity (list-feed-digest integration surface) ───────────── + +import { createHash } from 'node:crypto'; +import { assignStoryIdentity, deduplicateHeadlines } from '../server/worldmonitor/news/v1/dedup.mjs'; + +const sha256Hex = async (text) => createHash('sha256').update(text).digest('hex'); +const normalizeTitle = (title) => title.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, '').replace(/\s+/g, ' ').trim(); + +describe('assignStoryIdentity (#4919 acceptance)', () => { + it('REGRESSION: corroboration rises when the same event arrives with different wording', async () => { + // Under the old exact-hash identity these three wordings were three + // separate stories with corroborationCount=1 each. + const items = [ + { title: 'Iran threatens to close Strait of Hormuz if US blockade continues', source: 'Reuters' }, + { title: 'Iran threatens to close Strait of Hormuz — live updates', source: 'BBC' }, + { title: 'Iran threatens to close Strait of Hormuz if US blockade continues', source: 'AP' }, + { title: 'Stock market rallies on tech earnings report', source: 'CNBC' }, + ]; + const assignment = await assignStoryIdentity(items, normalizeTitle, sha256Hex); + assert.equal(assignment.get(items[0]).corroborationCount, 3, 'three sources, three wordings, ONE story'); + assert.equal(assignment.get(items[0]).titleHash, assignment.get(items[1]).titleHash); + assert.equal(assignment.get(items[0]).titleHash, assignment.get(items[2]).titleHash); + assert.equal(assignment.get(items[3]).corroborationCount, 1); + assert.notEqual(assignment.get(items[3]).titleHash, assignment.get(items[0]).titleHash); + }); + + it('singleton clusters hash exactly as the old identity (story:track keys unchanged)', async () => { + const items = [{ title: 'Kenya tax protests spread to Nairobi', source: 'AFP' }]; + const assignment = await assignStoryIdentity(items, normalizeTitle, sha256Hex); + assert.equal( + assignment.get(items[0]).titleHash, + await sha256Hex(normalizeTitle(items[0].title)), + ); + }); + + it('canonical hash is order-independent (stable across batch orderings)', async () => { + const a = { title: 'Iran threatens to close Strait of Hormuz — live updates', source: 'BBC' }; + const b = { title: 'Iran threatens to close Strait of Hormuz', source: 'Reuters' }; + const first = await assignStoryIdentity([a, b], normalizeTitle, sha256Hex); + const second = await assignStoryIdentity([b, a], normalizeTitle, sha256Hex); + assert.equal(first.get(a).titleHash, second.get(a).titleHash); + }); + + it('duplicate sources within a cluster count once', async () => { + const items = [ + { title: 'Turkey hikes interest rates to 50% in surprise move', source: 'Reuters' }, + { title: 'Turkey hikes rates to 50% in surprise move', source: 'Reuters' }, + ]; + const assignment = await assignStoryIdentity(items, normalizeTitle, sha256Hex); + assert.equal(assignment.get(items[0]).corroborationCount, 1, 'same outlet republishing is not corroboration'); + }); + + it('every item receives an assignment', async () => { + const items = [ + { title: 'A completely unique story about lunar mining', source: 'X' }, + { title: '', source: 'Y' }, + { title: '!!!', source: 'Z' }, + ]; + const assignment = await assignStoryIdentity(items, normalizeTitle, sha256Hex); + for (const item of items) assert.ok(assignment.get(item), `missing assignment for "${item.title}"`); + }); +}); + +describe('deduplicateHeadlines (shared-similarity rewrite)', () => { + it('keeps unvectorizable headlines rather than dropping them', () => { + const result = deduplicateHeadlines(['', '¡!', 'Fed holds rates steady']); + assert.equal(result.length, 3); + }); +}); + + +// ── #4924 review-round regression tests ──────────────────────────────────── + +describe('canonical identity stability (#4924 review)', () => { + it('REGRESSION: a later-published wording that sorts lexicographically earlier must NOT steal the canonical', async () => { + // Reliability P1 + learnings: pre-fix, canonical = lexicographic-min + // member of the CURRENT batch, so "Ahead of talks, Iran threatens…" + // (sorts before "Iran threatens…") joining a live cluster flipped the + // story:track id, resetting mentionCount/firstSeen and re-firing + // BREAKING mid-lifecycle. + const a = { title: 'Iran threatens to close Strait of Hormuz', source: 'Reuters', publishedAt: 100 }; + const b = { title: 'Iran threatens to close Strait of Hormuz — live updates', source: 'BBC', publishedAt: 200 }; + const build1 = await assignStoryIdentity([a, b], normalizeTitle, sha256Hex); + + const c = { title: 'Ahead of talks, Iran threatens to close Strait of Hormuz', source: 'AFP', publishedAt: 300 }; + const build2 = await assignStoryIdentity([a, b, c], normalizeTitle, sha256Hex); + + assert.ok(normalizeTitle(c.title) < normalizeTitle(a.title), 'fixture: C must sort before A for the test to bite'); + assert.equal(build2.get(a).titleHash, build1.get(a).titleHash, 'canonical must stay anchored on the earliest-published member'); + assert.equal(build2.get(c).titleHash, build1.get(a).titleHash, 'new wording adopts the existing identity'); + assert.equal(build2.get(a).corroborationCount, 3); + }); + + it('missing publishedAt falls back to lexicographic-min (deterministic)', async () => { + const a = { title: 'Iran threatens to close Strait of Hormuz — live updates', source: 'BBC' }; + const b = { title: 'Iran threatens to close Strait of Hormuz', source: 'Reuters' }; + const assignment = await assignStoryIdentity([a, b], normalizeTitle, sha256Hex); + assert.equal(assignment.get(a).titleHash, await sha256Hex(normalizeTitle(b.title))); + }); + + it('cluster membership is input-order independent (union-find, not greedy seed)', () => { + // Bridge case: A~B and B~C above threshold, A~C possibly below — + // greedy first-seed clustering could split this differently + // depending on which item arrived first; connected components + // cannot. + const texts = [ + 'Turkey central bank hikes interest rates to 50% in surprise move', + 'Turkey central bank hikes interest rates to 50%', + 'Turkey central bank hikes rates', + 'Kenya tax protests spread to Nairobi', + ]; + const perms = [ + [0, 1, 2, 3], [3, 2, 1, 0], [1, 3, 0, 2], [2, 0, 3, 1], + ]; + const canonicalSizes = perms.map((perm) => { + const clusters = clusterTexts(perm.map((i) => texts[i])); + return clusters.map((c) => c.length).sort().join(','); + }); + assert.ok(canonicalSizes.every((sig) => sig === canonicalSizes[0]), + `cluster size signature must be order-invariant, got: ${canonicalSizes.join(' | ')}`); + }); +}); + +describe('degenerate titles (#4924 review)', () => { + it('emoji/punctuation-only titles get per-item sentinel identities, not one shared phantom track', async () => { + const items = [ + { title: '🔥🔥🔥', source: 'FeedA' }, + { title: '!!!', source: 'FeedB' }, + { title: '🔥🔥🔥', source: 'FeedC' }, + ]; + const assignment = await assignStoryIdentity(items, normalizeTitle, sha256Hex); + const hashes = items.map((i) => assignment.get(i).titleHash); + assert.equal(new Set(hashes).size, 3, 'each contentless item gets its own identity'); + for (const item of items) { + assert.equal(assignment.get(item).corroborationCount, 1, 'contentless titles never corroborate each other'); + } + }); +}); + +describe('attribution suffixes and length clamp (#4924 review)', () => { + it('stripAttributionSuffix removes trailing publisher attributions', () => { + assert.equal(stripAttributionSuffix('Iran threatens Hormuz - Reuters'), 'Iran threatens Hormuz'); + assert.equal(stripAttributionSuffix('Iran threatens Hormuz - example.com'), 'Iran threatens Hormuz'); + assert.equal(stripAttributionSuffix('Iran-Iraq talks resume'), 'Iran-Iraq talks resume', 'mid-title hyphens untouched'); + }); + + it('a pathologically long title does not change identity behavior (clamped, still vectorizes)', () => { + const base = 'Iran threatens to close Strait of Hormuz'; + const long = base + ' ' + 'filler'.repeat(2000); + assert.ok(storyVector(long), 'clamped long title still vectorizes'); + assert.ok(storySimilarity(base, base + ' amid rising tension') >= STORY_SIMILARITY_THRESHOLD); + }); +}); + +// ── list-feed-digest integration wiring (source-textual, mirrors the +// digest-buildDigest-*-passthrough test pattern) ─────────────────────────── + +import { readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +describe('list-feed-digest story-identity wiring (#4924 review)', () => { + const digestSrc = readFileSync( + resolve(dirname(fileURLToPath(import.meta.url)), '../server/worldmonitor/news/v1/list-feed-digest.ts'), + 'utf-8', + ); + + it('assigns story identity before AI-cache enrichment and scoring', () => { + const assignAt = digestSrc.indexOf('await assignStoryIdentity(allItems'); + const enrichAt = digestSrc.indexOf('await enrichWithAiCache(allItems)'); + assert.ok(assignAt > -1 && enrichAt > -1); + assert.ok(assignAt < enrichAt, 'identity must be assigned before enrichment/scoring consumes it'); + }); + + it('both corroboration consumers read the cluster-wide count from the item', () => { + assert.match(digestSrc, /Math\.max\(item\.corroborationCount, item\.entityCorroborationCount\)/, + 'importance scoring must consume the cluster-wide corroboration'); + assert.match(digestSrc, /const sourceCount = item\.corroborationCount \?\? 1;/, + 'per-category slice must consume the cluster-wide corroboration'); + }); + + it('mentionCount increments once per unique hash per cycle, not once per member', () => { + const hincrbyAt = digestSrc.indexOf("['HINCRBY', trackKey, 'mentionCount', '1']"); + const guardAt = digestSrc.indexOf('if (!writtenHashes.has(hash))'); + assert.ok(guardAt > -1, 'unique-hash guard must exist'); + assert.ok(hincrbyAt > guardAt, 'HINCRBY must sit inside the once-per-hash guard'); + // Per-member set-shaped writes stay outside the guard. + const saddAt = digestSrc.indexOf("['SADD', sourcesKey, item.source]"); + assert.ok(saddAt > hincrbyAt, 'per-member SADD stays outside the once-per-hash block'); + }); + + it('coverage-miss fallback is observable, not silent', () => { + assert.match(digestSrc, /story-identity coverage miss/, 'fallback branch must log'); + }); +}); + +describe('hot-bucket mega-story pre-union (#4924 external review)', () => { + it('251 identical titles cluster together even when every token bucket is hot', () => { + const texts = Array.from({ length: 251 }, () => 'Iran threatens to close Strait of Hormuz'); + texts.push('Kenya tax protests spread to Nairobi'); + const clusters = clusterTexts(texts); + const sizes = clusters.map((c) => c.length).sort((a, b) => b - a); + assert.equal(sizes[0], 251, 'identical titles must union regardless of bucket caps'); + assert.equal(sizes[1], 1); + }); +}); + +// ── #4924 external-review round: cross-cycle continuity + TTL ordering ───── + +import { adoptExistingCanonical } from '../server/worldmonitor/news/v1/dedup.mjs'; + +const sha256HexNode = sha256Hex; +const normalizeBasic = normalizeTitle; + +describe('cross-cycle canonical adoption (#4924 external review P1)', () => { + it('REGRESSION: two-cycle A+B -> B-only keeps the story identity via alias adoption', async () => { + const A = { title: 'Iran threatens to close Strait of Hormuz', source: 'Reuters', publishedAt: 1_000 }; + const B = { title: 'Iran warns it may close the Strait of Hormuz', source: 'BBC', publishedAt: 2_000 }; + + // Cycle 1: A+B cluster; A (earlier publishedAt) anchors the canonical. + const cycle1 = await assignStoryIdentity([A, B], normalizeBasic, sha256HexNode); + const identity1 = cycle1.get(A); + assert.equal(identity1.titleHash, cycle1.get(B).titleHash, 'A and B share one identity in cycle 1'); + assert.ok(identity1.memberTitleHashes.length >= 2, 'both member hashes exposed for alias persistence'); + + // The digest persists memberHash -> canonical alias rows; simulate them. + const aliasMap = new Map(identity1.memberTitleHashes.map((h) => [h, identity1.titleHash])); + + // Cycle 2: only B appears — batch-derived canonical would be B itself... + const cycle2 = await assignStoryIdentity([{ ...B }], normalizeBasic, sha256HexNode); + const identity2 = [...cycle2.values()][0]; + assert.notEqual(identity2.titleHash, identity1.titleHash, 'without adoption, B forks a fresh identity'); + + // ...but adoption re-anchors to the live canonical from cycle 1. + const adopted = adoptExistingCanonical(identity2.memberTitleHashes, identity2.titleHash, aliasMap); + assert.equal(adopted, identity1.titleHash, 'B-only cycle must continue the cycle-1 story track'); + }); + + it('adoption is deterministic: most-common live target wins, ties break lexicographically', () => { + const aliasMap = new Map([['m1', 'hash-b'], ['m2', 'hash-a'], ['m3', 'hash-b']]); + assert.equal(adoptExistingCanonical(['m1', 'm2', 'm3'], 'default', aliasMap), 'hash-b'); + const tied = new Map([['m1', 'hash-b'], ['m2', 'hash-a']]); + assert.equal(adoptExistingCanonical(['m1', 'm2'], 'default', tied), 'hash-a', 'tie -> smallest hash'); + assert.equal(adoptExistingCanonical(['m1'], 'default', new Map()), 'default', 'no live alias -> batch canonical'); + assert.equal(adoptExistingCanonical(undefined, 'default', aliasMap), 'default'); + }); +}); + +describe('story key TTL ordering (#4924 external review P2)', () => { + it('EXPIRE for sources/peak keys is queued with the per-member creating writes, never before them', () => { + const src = readFileSync( + resolve(dirname(fileURLToPath(import.meta.url)), '../server/worldmonitor/news/v1/list-feed-digest.ts'), + 'utf-8', + ); + const onceBlock = src.slice(src.indexOf("['HINCRBY', trackKey"), src.indexOf("['ZADD', peakKey, 'GT'")); + assert.ok(!onceBlock.includes("['EXPIRE', sourcesKey"), + 'sources EXPIRE must not sit in the once-per-hash pre-block — EXPIRE on a missing key is a no-op and the later SADD creates a persistent key'); + assert.ok(!onceBlock.includes("['EXPIRE', peakKey"), + 'peak EXPIRE must not sit in the once-per-hash pre-block'); + const memberBlock = src.slice(src.indexOf("['ZADD', peakKey, 'GT'"), src.indexOf('runRedisPipeline(commands)')); + assert.ok(memberBlock.includes("['EXPIRE', sourcesKey") && memberBlock.includes("['EXPIRE', peakKey"), + 'both EXPIREs must follow the creating SADD/ZADD in the per-member block'); + assert.match(src, /STORY_ALIAS_KEY\(memberHash\), hash, 'EX', ttl/, 'alias rows persisted with story TTL'); + assert.match(src, /adoptExistingCanonical\(identity\.memberTitleHashes, identity\.titleHash, aliasTargetByHash\)/, + 'digest must adopt live canonicals before assigning hashes'); + }); +});