Externalize and expand headline aliases

Move the hard-coded ALIASES dict into resources/aliases.toml (loaded with
stdlib tomllib), expand it with agencies, international bodies, and country
names, and fix two correctness bugs in the naive replace():

- Whole-word matching so 'united states' no longer collapses inside
  'united statesman', and 'inflation rate' no longer mangles 'inflation rates'.
- Longest-phrase-first application so 'president of the united states'
  resolves to 'potus' before 'united states' fires.
- Correct a stopword collision: 'united nations' -> 'un' was silently dropped
  because 'un' is a stopword; it now maps to 'unitednations'.

Aliases are threaded through normalize_headline -> is_headline ->
_build_headlines -> prepare_headlines and loaded in main.py. Includes unit
tests and README coverage.

README.md

@@ -14,7 +14,8 @@ question: *which stories are multiple independent sources reporting right now?*
    are first checked for embedded JSON (JSON-LD / Next.js SSR state) before
    falling back to scraping `<a>`/`<span>` text, associating each headline with
    the nearest `<time>` publication timestamp.
-3. **Normalize** each headline (lowercase, strip punctuation, remove stopwords).
+3. **Normalize** each headline (lowercase, strip punctuation, apply aliases so
+   "Federal Reserve" becomes "fed", then remove stopwords).
 4. **Cluster** headlines into stories using pairwise cosine similarity, linking
    matches transitively so a chain of near-duplicates collapses into one story.
 5. **Filter & report** stories with at least `--min-sources` distinct outlets,
@@ -45,8 +46,9 @@ Run from the project root — the resource paths are relative (`./resources/...`
   feeds; `#` comments and blank lines are ignored).
 - `resources/stopwords.txt` — one stopword per line.
 - `resources/excluded_phrases.txt` — boilerplate link text to ignore (see below).
-- Defaults live at the top of `main.py` (`SIMILARITY_THRESHOLD = 0.75`,
-  `MIN_SOURCES = 2`) and can be overridden on the command line.
+- `resources/aliases.toml` — full-name → short-form aliases (see below).
+- Defaults live at the top of `main.py` (`SIMILARITY_THRESHOLD = 0.64`,
+  `MIN_SOURCES = 3`) and can be overridden on the command line.
 
 ## Usage
 
@@ -56,8 +58,8 @@ python main.py [options]
 
 | Option | Description | Default |
 | --- | --- | --- |
-| `--min-sources N` | Only output stories reported by at least N distinct sources | `2` |
-| `--threshold T` | Cosine similarity used to consider two headlines the same story | `0.75` |
+| `--min-sources N` | Only output stories reported by at least N distinct sources | `3` |
+| `--threshold T` | Cosine similarity used to consider two headlines the same story | `0.64` |
 | `--since YYYY-MM-DD` | Only consider headlines published on or after this date | *(none)* |
 | `--until YYYY-MM-DD` | Only consider headlines published on or before this date | *(none)* |
 | `--verbose` | Enable debug logging | off |
@@ -78,7 +80,7 @@ python main.py --since 2026-09-01 --until 2026-09-14 --threshold 0.8
 Sample output:
 
 ```
-News stories covered by at least 2 distinct sources (published on or after 2026-09-13)
+News stories covered by at least 3 distinct sources (published on or after 2026-09-13)
 ============================================================
 
 [3 source(s)] Federal Reserve holds interest rates steady (2026-09-14)
@@ -116,6 +118,13 @@ News stories covered by at least 2 distinct sources (published on or after 2026-
   `excluded_phrases.txt` (case- and punctuation-insensitive; an entry matches when
   it appears anywhere in the link text as a run of words, so `privacy choices`
   also catches "Your Privacy Choices"). Extend the file rather than editing code.
+- **Aliases unify equivalent names.** `aliases.toml` maps full names to a shared
+  short form so "Federal Reserve" and "the Fed" normalize to the same token and
+  cluster. Matching is whole-word only ("united states" won't match inside
+  "united statesman") and longest-phrase-first ("president of the united states"
+  becomes "potus" before "united states" can fire). The canonical form must not be
+  a stopword — e.g. "united nations" uses `unitednations`, not `un`, because `un`
+  is stripped during normalization. Extend the file rather than editing code.
 
 ## Tests
 
@@ -132,7 +141,7 @@ anya/
 │   ├── headlines.py        # fetch + parse headlines (and timestamps)
 │   ├── feeds.py            # RSS/Atom feed detection and parsing
 │   ├── ssr.py              # JSON-LD / Next.js embedded-JSON extraction
-│   ├── normalization.py    # stopword/phrase loading and headline normalization
+│   ├── normalization.py    # stopword/alias/phrase loading and headline normalization
 │   ├── similarity.py       # cosine similarity over token lists
 │   ├── sources.py          # load source URLs
 │   ├── stories.py          # cluster headlines into stories
@@ -143,6 +152,7 @@ anya/
 ├── resources/
 │   ├── sources.txt            # one source URL per line
 │   ├── stopwords.txt          # one stopword per line
+│   ├── aliases.toml           # full-name → short-form aliases
 │   └── excluded_phrases.txt   # boilerplate link text to ignore
 └── tests/                  # unit tests
 ```

main.py

@@ -4,7 +4,7 @@ from datetime import date
 
 from services.dates import filter_by_date_window, parse_date_arg
 from services.headlines import prepare_headlines
-from services.normalization import get_excluded_phrases, get_stopwords
+from services.normalization import get_aliases, get_excluded_phrases, get_stopwords
 from services.sources import get_sources
 from services.stories import cluster_stories
 
@@ -17,6 +17,7 @@ logger = logging.getLogger(__name__)
 SOURCE_FILE = './resources/sources.txt'
 STOPWORDS_FILE = './resources/stopwords.txt'
 EXCLUDED_PHRASES_FILE = './resources/excluded_phrases.txt'
+ALIASES_FILE = './resources/aliases.toml'
 SIMILARITY_THRESHOLD = 0.64
 MIN_SOURCES = 3
 
@@ -95,9 +96,17 @@ def main():
         logger.critical("Failed to load excluded phrases from '%s': %s", EXCLUDED_PHRASES_FILE, e, exc_info=True)
         return
 
+    try:
+        logger.info("Loading aliases from '%s'", ALIASES_FILE)
+        aliases = get_aliases(ALIASES_FILE)
+        logger.info("Loaded %d aliases successfully", len(aliases))
+    except Exception as e:
+        logger.critical("Failed to load aliases from '%s': %s", ALIASES_FILE, e, exc_info=True)
+        return
+
     try:
         logger.info("Fetching and preparing headlines from %d sources", len(sources))
-        headlines = prepare_headlines(sources, stopwords, excluded_phrases=excluded_phrases)
+        headlines = prepare_headlines(sources, stopwords, excluded_phrases=excluded_phrases, aliases=aliases)
         logger.info("Total prepared headlines available for comparison: %d", len(headlines))
     except Exception as e:
         logger.critical("Failed during headline preparation: %s", e, exc_info=True)

resources/aliases.toml

@@ -0,0 +1,95 @@
+# Headline aliases — maps a full name/phrase to a canonical short form so that
+# "Federal Reserve" and "the Fed" normalize to the same token and cluster together.
+#
+# Format: one entry per line under [aliases]:
+#     "full phrase" = "canonical"
+#
+# Matching rules:
+#   * Case-insensitive and after punctuation removal (so "people's" == "peoples").
+#   * Whole-word only — "united states" will not match inside "united statesman".
+#   * Longest phrases are applied first — "president of the united states" wins
+#     over "united states".
+#   * The canonical form MUST NOT be a stopword (see resources/stopwords.txt), or
+#     it will be stripped during normalization and the alias is lost.
+
+[aliases]
+
+# ---------------------------------------------------------------- US government
+"federal reserve" = "fed"
+"federal reserve system" = "fed"
+"white house" = "wh"
+"central intelligence agency" = "cia"
+"federal bureau of investigation" = "fbi"
+"national security agency" = "nsa"
+"department of justice" = "doj"
+"department of defense" = "dod"
+"department of homeland security" = "dhs"
+"environmental protection agency" = "epa"
+"securities and exchange commission" = "sec"
+"internal revenue service" = "irs"
+"social security administration" = "ssa"
+"centers for disease control" = "cdc"
+"centers for disease control and prevention" = "cdc"
+"food and drug administration" = "fda"
+"federal aviation administration" = "faa"
+"federal communications commission" = "fcc"
+"federal trade commission" = "ftc"
+"federal emergency management agency" = "fema"
+"national aeronautics and space administration" = "nasa"
+"national institutes of health" = "nih"
+"national transportation safety board" = "ntsb"
+"supreme court" = "scotus"
+"supreme court of the united states" = "scotus"
+"house of representatives" = "house"
+"president of the united states" = "potus"
+"vice president" = "vp"
+"prime minister" = "pm"
+"secretary of state" = "secstate"
+"attorney general" = "ag"
+
+# ---------------------------------------------------------- International bodies
+# "un" and "who" are stopwords, so these use concatenated forms to survive.
+"united nations" = "unitednations"
+"world health organization" = "worldhealthorg"
+"united nations security council" = "unsc"
+"international monetary fund" = "imf"
+"world trade organization" = "wto"
+"north atlantic treaty organization" = "nato"
+"international criminal court" = "icc"
+"international atomic energy agency" = "iaea"
+"organization of the petroleum exporting countries" = "opec"
+"european union" = "eu"
+"african union" = "au"
+"association of southeast asian nations" = "asean"
+
+# ----------------------------------------------------------- Countries & regions
+"united states" = "us"
+"united states of america" = "us"
+"united kingdom" = "uk"
+"great britain" = "uk"
+"united arab emirates" = "uae"
+"people's republic of china" = "china"
+"republic of korea" = "south korea"
+"democratic people's republic of korea" = "north korea"
+"saudi arabia" = "saudi"
+"middle east" = "mideast"
+"asia pacific" = "apac"
+"latin america" = "latam"
+
+# ----------------------------------------------------------- Politics & economy
+"republican party" = "gop"
+"democratic party" = "democrats"
+"gross domestic product" = "gdp"
+"gross national product" = "gnp"
+"consumer price index" = "cpi"
+"initial public offering" = "ipo"
+"unemployment rate" = "unemployment"
+"inflation rate" = "inflation"
+"national debt" = "debt"
+"budget deficit" = "deficit"
+
+# ---------------------------------------------------------- Technology & other
+"artificial intelligence" = "ai"
+"machine learning" = "ml"
+"electric vehicle" = "ev"
+"international space station" = "iss"

services/headlines.py

@@ -88,22 +88,22 @@ def _extract_candidates(source_content):
     return candidates
 
 
-def _build_headlines(items, source_url, stopwords, excluded_phrases):
+def _build_headlines(items, source_url, stopwords, excluded_phrases, aliases=None):
     """Turn ``(title, published_at)`` tuples into headline objects, applying filters."""
     collected = []
     for title, published_at in items:
         cleaned_text = title.strip()
-        if not cleaned_text or not is_headline(cleaned_text, stopwords, excluded_phrases):
+        if not cleaned_text or not is_headline(cleaned_text, stopwords, excluded_phrases, aliases):
             continue
         try:
-            normalized = normalize_headline(cleaned_text, stopwords)
+            normalized = normalize_headline(cleaned_text, stopwords, aliases)
             collected.append(Headline(cleaned_text, normalized, source_url, published_at))
         except Exception as e:
             logger.error("Failed to normalize title %r from '%s': %s", cleaned_text, source_url, e, exc_info=True)
     return collected
 
 
-def is_headline(text, stopwords=None, excluded_phrases=None):
+def is_headline(text, stopwords=None, excluded_phrases=None, aliases=None):
     if not text or not isinstance(text, str):
         return False
     cleaned_text = text.strip()
@@ -120,14 +120,14 @@ def is_headline(text, stopwords=None, excluded_phrases=None):
         logger.debug("Text rejected as non-headline (matches excluded phrase): %r", cleaned_text)
         return False
     if stopwords is not None:
-        normalized = normalize_headline(cleaned_text, stopwords)
+        normalized = normalize_headline(cleaned_text, stopwords, aliases)
         if not normalized:
             logger.debug("Text rejected as headline (no meaningful tokens after stopword removal): %r", cleaned_text)
             return False
     return True
 
 
-def prepare_headlines(sources, stopwords, timeout=DEFAULT_TIMEOUT, headers=None, excluded_phrases=None):
+def prepare_headlines(sources, stopwords, timeout=DEFAULT_TIMEOUT, headers=None, excluded_phrases=None, aliases=None):
     request_headers = headers if headers is not None else DEFAULT_HEADERS
     logger.info("Starting preparation of headlines for %d sources", len(sources) if sources else 0)
     headlines = []
@@ -162,7 +162,7 @@ def prepare_headlines(sources, stopwords, timeout=DEFAULT_TIMEOUT, headers=None,
         if looks_like_feed(source_content):
             logger.info("Detected RSS/Atom feed for source '%s'", source_url)
             try:
-                feed_headlines = _build_headlines(parse_feed(source_content), source_url, stopwords, excluded_phrases)
+                feed_headlines = _build_headlines(parse_feed(source_content), source_url, stopwords, excluded_phrases, aliases)
             except Exception as e:
                 logger.error("Failed to parse feed from '%s': %s", source_url, e, exc_info=True)
                 continue
@@ -176,7 +176,7 @@ def prepare_headlines(sources, stopwords, timeout=DEFAULT_TIMEOUT, headers=None,
             logger.error("Failed to extract embedded JSON from '%s': %s", source_url, e, exc_info=True)
             ssr_items = []
         if ssr_items:
-            ssr_headlines = _build_headlines(ssr_items, source_url, stopwords, excluded_phrases)
+            ssr_headlines = _build_headlines(ssr_items, source_url, stopwords, excluded_phrases, aliases)
             headlines.extend(ssr_headlines)
             logger.info("Successfully extracted %d headlines via embedded JSON from source '%s'", len(ssr_headlines), source_url)
             continue
@@ -195,12 +195,12 @@ def prepare_headlines(sources, stopwords, timeout=DEFAULT_TIMEOUT, headers=None,
             if not cleaned_text:
                 logger.debug("Skipping empty tag text at position %d from '%s'", tag_idx, source_url)
                 continue
-            if not is_headline(cleaned_text, stopwords, excluded_phrases):
+            if not is_headline(cleaned_text, stopwords, excluded_phrases, aliases):
                 logger.debug("Skipping non-headline tag text at position %d from '%s': %r", tag_idx, source_url, cleaned_text)
                 continue
             logger.debug("Processing tag [%d/%d] from '%s': %r", tag_idx, len(candidates), source_url, cleaned_text)
             try:
-                normalized_headline = normalize_headline(cleaned_text, stopwords)
+                normalized_headline = normalize_headline(cleaned_text, stopwords, aliases)
                 headline = Headline(cleaned_text, normalized_headline, source_url, published_at)
                 headlines.append(headline)
                 source_headlines_count += 1

services/normalization.py

@@ -1,47 +1,49 @@
 import logging
+import re
 import string
+import tomllib
 
 logger = logging.getLogger(__name__)
 
-ALIASES = {
-    'federal reserve': 'fed',
-    'united states': 'us',
-    'united kingdom': 'uk',
-    'european union': 'eu',
-    'north atlantic treaty organization': 'nato',
-    'white house': 'wh',
-    'united nations': 'un',
-    'international monetary fund': 'imf',
-    'world trade organization': 'wto',
-    'central intelligence agency': 'cia',
-    'federal bureau of investigation': 'fbi',
-    'department of justice': 'doj',
-    'department of defense': 'dod',
-    'environmental protection agency': 'epa',
-    'securities and exchange commission': 'sec',
-    'internal revenue service': 'irs',
-    'social security administration': 'ssa',
-    'centers for disease control': 'cdc',
-    'national security agency': 'nsa',
-    'department of homeland security': 'dhs',
-    'supreme court': 'scotus',
-    'republican party': 'gop',
-    'democratic party': 'democrats',
-    'president of the united states': 'potus',
-    'vice president': 'vp',
-    'prime minister': 'pm',
-    'secretary of state': 'secstate',
-    'attorney general': 'ag',
-    'gross domestic product': 'gdp',
-    'consumer price index': 'cpi',
-    'unemployment rate': 'unemployment',
-    'inflation rate': 'inflation',
-    'national debt': 'debt',
-    'budget deficit': 'deficit',
-    'middle east': 'mideast',
-    'asia pacific': 'apac',
-    'latin america': 'latam'
-}
+
+def get_aliases(path):
+    """Load headline aliases from a TOML file with an ``[aliases]`` table.
+
+    Each ``"full phrase" = "canonical"`` entry maps a multi-word name to a
+    canonical short form (e.g. ``"federal reserve" = "fed"``). Keys are
+    lowercased, punctuation-stripped, and whitespace-collapsed to match the
+    text form that ``normalize_headline`` applies aliases to.
+    """
+    logger.info("Attempting to load aliases from file: '%s'", path)
+    try:
+        with open(path, 'rb') as aliases_file:
+            data = tomllib.load(aliases_file)
+    except FileNotFoundError:
+        logger.error("Aliases file not found at path: '%s'", path, exc_info=True)
+        raise
+    except PermissionError:
+        logger.error("Permission denied accessing aliases file: '%s'", path, exc_info=True)
+        raise
+    except tomllib.TOMLDecodeError as e:
+        logger.error("Failed to parse aliases file '%s': %s", path, e, exc_info=True)
+        raise
+    except Exception as e:
+        logger.error("Failed to load aliases from '%s': %s", path, e, exc_info=True)
+        raise
+
+    aliases = {}
+    for phrase, canonical in data.get('aliases', {}).items():
+        # Mirror normalize_headline's pre-alias text form: lowercase, then
+        # strip punctuation (not space-replace), then collapse whitespace.
+        phrase = str(phrase).strip().lower()
+        for char in string.punctuation:
+            phrase = phrase.replace(char, '')
+        phrase = ' '.join(phrase.split())
+        canonical = str(canonical).strip().lower()
+        if phrase and canonical:
+            aliases[phrase] = canonical
+    logger.info("Successfully loaded %d aliases from '%s'", len(aliases), path)
+    return aliases
 
 
 def get_stopwords(path):
@@ -150,22 +152,32 @@ def remove_stopwords(text, stopwords):
     return sentence_words
 
 
-def normalize_headline(text, stopwords):
+def replace_aliases(text, aliases):
+    """Swap full phrases for canonical forms, longest first, whole-word only.
+
+    Longer phrases are applied before shorter ones so ``president of the united
+    states`` becomes ``potus`` rather than leaving ``united states`` for the
+    ``us`` rule to snag. ``\b`` boundaries prevent substring matches such as
+    collapsing ``united statesman`` to ``usman``.
+    """
+    if not aliases:
+        return text
+    text = ' '.join(text.split())
+    for phrase in sorted(aliases, key=len, reverse=True):
+        pattern = re.compile(r'\b' + re.escape(phrase) + r'\b')
+        text = pattern.sub(aliases[phrase], text)
+    return text
+
+
+def normalize_headline(text, stopwords, aliases=None):
     logger.debug("Starting normalization of headline: %r", text)
     headline = text.strip().lower()
     punctuation = string.punctuation
     for char in punctuation:
         headline = headline.replace(char, '')
     logger.debug("Headline after lowercasing and punctuation removal: %r", headline)
-    headline = replace_common_aliases(headline)
-    logger.debug("Headline after replacing common aliases: %r", headline)
+    headline = replace_aliases(headline, aliases)
+    logger.debug("Headline after replacing aliases: %r", headline)
     normalized = remove_stopwords(headline, stopwords)
     logger.debug("Completed normalization for %r -> %s", text, normalized)
     return normalized
-
-
-def replace_common_aliases(headline):
-    for alias in ALIASES:
-        headline = headline.replace(alias, ALIASES[alias])
-
-    return headline

tests/test_aliases.py

@@ -0,0 +1,109 @@
+import os
+import tempfile
+import unittest
+
+from services.normalization import get_aliases, normalize_headline, replace_aliases
+
+
+ALIASES = {
+    "federal reserve": "fed",
+    "united states": "us",
+    "united states of america": "us",
+    "president of the united states": "potus",
+    "united nations": "unitednations",
+    "inflation rate": "inflation",
+}
+
+
+class TestGetAliases(unittest.TestCase):
+    def test_loads_toml_with_key_normalization(self):
+        path = None
+        try:
+            with tempfile.NamedTemporaryFile("wb", suffix=".toml", delete=False) as f:
+                f.write(
+                    b"[aliases]\n"
+                    b"\"Federal Reserve\" = \"Fed\"\n"
+                    b"\"people's republic of china\" = \"china\"\n"
+                )
+                path = f.name
+            aliases = get_aliases(path)
+            self.assertEqual(aliases["federal reserve"], "fed")
+            # apostrophe is stripped so keys match punctuation-removed text
+            self.assertEqual(aliases["peoples republic of china"], "china")
+        finally:
+            if path:
+                os.unlink(path)
+
+    def test_missing_file_raises(self):
+        with self.assertRaises(FileNotFoundError):
+            get_aliases("/no/such/aliases.toml")
+
+
+class TestReplaceAliases(unittest.TestCase):
+    def test_whole_word_only(self):
+        # "united states" must not match inside "united statesman".
+        self.assertEqual(
+            replace_aliases("united statesman wins award", ALIASES),
+            "united statesman wins award",
+        )
+
+    def test_longest_phrase_first(self):
+        # "president of the united states" -> "potus", not "president of the us".
+        self.assertEqual(
+            replace_aliases("president of the united states speaks", ALIASES),
+            "potus speaks",
+        )
+
+    def test_no_aliases_returns_text(self):
+        self.assertEqual(replace_aliases("plain text here", None), "plain text here")
+        self.assertEqual(replace_aliases("plain text here", {}), "plain text here")
+
+
+class TestNormalizeHeadlineAliases(unittest.TestCase):
+    def setUp(self):
+        self.stopwords = {"the", "a", "an", "in", "on", "and", "of", "to", "for"}
+
+    def test_full_name_and_acronym_cluster(self):
+        # "Federal Reserve" and "the Fed" normalize to the same first token.
+        full = normalize_headline("Federal Reserve raises rates", self.stopwords, ALIASES)
+        short = normalize_headline("the Fed raises rates", self.stopwords, ALIASES)
+        self.assertEqual(full, ["fed", "raises", "rates"])
+        self.assertEqual(short, ["fed", "raises", "rates"])
+
+    def test_united_nations_not_dropped(self):
+        # Regression guard: the canonical must survive stopword removal.
+        self.assertEqual(
+            normalize_headline("United Nations meets", self.stopwords, ALIASES),
+            ["unitednations", "meets"],
+        )
+
+    def test_inflation_rates_not_mangled(self):
+        # "inflation rate" must not fire inside "inflation rates".
+        self.assertEqual(
+            normalize_headline("inflation rates fall", self.stopwords, ALIASES),
+            ["inflation", "rates", "fall"],
+        )
+
+
+class TestAliasesFileIntegrity(unittest.TestCase):
+    """Guard against an alias canonical colliding with a stopword, which would
+    get silently stripped during normalization (the original 'united nations'
+    -> 'un' bug)."""
+
+    def test_no_canonical_is_a_stopword(self):
+        base = os.path.join(os.path.dirname(__file__), "..", "resources")
+        aliases = get_aliases(os.path.join(base, "aliases.toml"))
+        with open(os.path.join(base, "stopwords.txt"), encoding="utf-8-sig") as f:
+            stopwords = set(f.read().splitlines())
+        self.assertTrue(aliases)
+        for phrase, canonical in aliases.items():
+            for token in canonical.split():
+                self.assertNotIn(
+                    token,
+                    stopwords,
+                    "alias %r canonical token %r is a stopword" % (phrase, token),
+                )
+
+
+if __name__ == "__main__":
+    unittest.main()