From b15bc584526b255e3e6fc20bd09a91d498cc7205 Mon Sep 17 00:00:00 2001 From: Jared Bell Date: Tue, 15 Sep 2026 00:05:26 +0000 Subject: [PATCH] Add RSS/Atom support, expand sources and excluded phrases - services/feeds.py parses RSS 2.0/1.0 and Atom into (title, date) entries - prepare_headlines auto-detects feeds by content and parses them directly - strip feeds./rss./moxie. subdomains so feeds collapse to the outlet domain - add 7 verified RSS feeds (BBC, Guardian, Al Jazeera, WaPo, The Hill, Vox, CNBC) - get_sources skips blank lines and # comments - broaden excluded-phrase list (skip links, share/follow, newsletter, utility) --- README.md | 19 ++++-- resources/excluded_phrases.txt | 47 ++++++++++++--- resources/sources.txt | 10 +++- services/feeds.py | 65 +++++++++++++++++++++ services/headlines.py | 27 +++++++++ services/sources.py | 25 ++++---- structs/headline.py | 14 +++-- tests/test_feeds.py | 103 +++++++++++++++++++++++++++++++++ 8 files changed, 281 insertions(+), 29 deletions(-) create mode 100644 services/feeds.py create mode 100644 tests/test_feeds.py diff --git a/README.md b/README.md index 585956a..4725e1c 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,9 @@ question: *which stories are multiple independent sources reporting right now?* ## How it works 1. **Load** source URLs and stopwords from `resources/`. -2. **Fetch** each source and extract candidate headline text from ``/`` - tags, associating each headline with the nearest ``/`` text, + associating each headline with the nearest `Story")) + + def test_empty_not_detected(self): + self.assertFalse(looks_like_feed("")) + self.assertFalse(looks_like_feed(None)) + + +class TestParseFeed(unittest.TestCase): + def test_rss_2_0(self): + items = parse_feed(RSS_2_0) + titles = [t for t, _ in items] + self.assertEqual(titles, ["First news story", "Second news story"]) + self.assertEqual(items[0][1].year, 2026) + self.assertIsNone(items[1][1]) + + def test_atom(self): + items = parse_feed(ATOM) + titles = [t for t, _ in items] + self.assertEqual(titles, ["Atom story one", "Atom story two"]) + self.assertIsNotNone(items[0][1]) + self.assertIsNotNone(items[1][1]) + + def test_malformed_returns_empty(self): + self.assertEqual(parse_feed("not xml at all <<<"), []) + + +class TestGetSources(unittest.TestCase): + def test_skips_comments_and_blanks(self): + path = None + try: + with tempfile.NamedTemporaryFile("w", suffix=".txt", delete=False) as f: + f.write("# a comment\n\nhttps://a.com\n\nhttps://b.com\n# another\n") + path = f.name + self.assertEqual(get_sources(path), ["https://a.com", "https://b.com"]) + finally: + if path: + os.unlink(path) + + +class TestFeedDomainNormalization(unittest.TestCase): + def test_feed_subdomain_stripped(self): + self.assertEqual(Headline("a", ["a"], "https://feeds.npr.org/1001/rss.xml").domain, "npr.org") + self.assertEqual(Headline("a", ["a"], "https://rss.nytimes.com/x").domain, "nytimes.com") + self.assertEqual(Headline("a", ["a"], "https://www.vox.com/rss/index.xml").domain, "vox.com") + + +class TestPrepareHeadlinesFeed(unittest.TestCase): + def setUp(self): + self.stopwords = {"the", "a", "an", "in", "on", "and", "of", "to"} + + @patch("services.headlines.requests.get") + def test_feed_source_is_parsed(self, mock_get): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.content = RSS_2_0.encode("utf-8") + mock_response.text = RSS_2_0 + mock_get.return_value = mock_response + + headlines = prepare_headlines(["https://feeds.bbci.co.uk/news/world/rss.xml"], self.stopwords) + + titles = [h.display_text for h in headlines] + self.assertEqual(titles, ["First news story", "Second news story"]) + # Domain comes from the feed host, normalized to the outlet. + self.assertEqual(headlines[0].domain, "bbci.co.uk") + self.assertIsNotNone(headlines[0].published_at) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file