Normalize headlines with common aliases, adjust threshold to 0.64, and expand RSS sources list.

main.py

@@ -17,7 +17,7 @@ logger = logging.getLogger(__name__)
 SOURCE_FILE = './resources/sources.txt'
 STOPWORDS_FILE = './resources/stopwords.txt'
 EXCLUDED_PHRASES_FILE = './resources/excluded_phrases.txt'
-SIMILARITY_THRESHOLD = 0.68
+SIMILARITY_THRESHOLD = 0.64
 MIN_SOURCES = 3
 
 # Used so stories without a parsable date sort behind dated ones.

resources/sources.txt

@@ -36,9 +36,13 @@ https://www.philstar.com/rss/world
 https://www.thejakartapost.com/
 
 # Policy & national
-https://feeds.washingtonpost.com/rss/world
+# https://feeds.washingtonpost.com/rss/world
 https://thehill.com/feed/
 https://www.vox.com/rss/index.xml
+https://feeds.feedburner.com/breitbart
+https://justthenews.com/rss.xml
+https://www.usatoday.com/
+https://feeds.skynews.com/feeds/rss/us.xml
 
 # Business
 https://www.cnbc.com/id/100003114/device/rss/rss.html
@@ -48,3 +52,8 @@ https://www.forbes.com/business/
 https://theintercept.com/feed/
 https://www.occrp.org/en
 https://www.bellingcat.com/
+https://www.icij.org/feed/
+https://www.propublica.org/feeds/propublica/main
+https://forbiddenstories.org/feed/
+https://www.themarshallproject.org/rss/recent
+https://www.yahoo.com/news/

services/normalization.py

@@ -3,6 +3,45 @@ import string
 
 logger = logging.getLogger(__name__)
 
+ALIASES = {
+    'federal reserve': 'fed',
+    'united states': 'us',
+    'united kingdom': 'uk',
+    'european union': 'eu',
+    'north atlantic treaty organization': 'nato',
+    'united nations': 'un',
+    'international monetary fund': 'imf',
+    'world trade organization': 'wto',
+    'central intelligence agency': 'cia',
+    'federal bureau of investigation': 'fbi',
+    'department of justice': 'doj',
+    'department of defense': 'dod',
+    'environmental protection agency': 'epa',
+    'securities and exchange commission': 'sec',
+    'internal revenue service': 'irs',
+    'social security administration': 'ssa',
+    'centers for disease control': 'cdc',
+    'national security agency': 'nsa',
+    'department of homeland security': 'dhs',
+    'supreme court': 'scotus',
+    'republican party': 'gop',
+    'democratic party': 'democrats',
+    'president of the united states': 'potus',
+    'vice president': 'vp',
+    'prime minister': 'pm',
+    'secretary of state': 'secstate',
+    'attorney general': 'ag',
+    'gross domestic product': 'gdp',
+    'consumer price index': 'cpi',
+    'unemployment rate': 'unemployment',
+    'inflation rate': 'inflation',
+    'national debt': 'debt',
+    'budget deficit': 'deficit',
+    'middle east': 'mideast',
+    'asia pacific': 'apac',
+    'latin america': 'latam'
+}
+
 
 def get_stopwords(path):
     logger.info("Attempting to load stopwords from file: '%s'", path)
@@ -117,6 +156,15 @@ def normalize_headline(text, stopwords):
     for char in punctuation:
         headline = headline.replace(char, '')
     logger.debug("Headline after lowercasing and punctuation removal: %r", headline)
+    headline = replace_common_aliases(headline)
+    logger.debug("Headline after replacing common aliases: %r", headline)
     normalized = remove_stopwords(headline, stopwords)
     logger.debug("Completed normalization for %r -> %s", text, normalized)
-    return normalized
+    return normalized
+
+
+def replace_common_aliases(headline):
+    for alias in ALIASES:
+        headline = headline.replace(alias, ALIASES[alias])
+
+    return headline