jbell730
/
anya


			
							import os
import tempfile
import unittest
from unittest.mock import patch, MagicMock

from services.headlines import is_headline, prepare_headlines
from services.normalization import canonicalize, get_excluded_phrases, is_excluded


PHRASES = [
    "skip to content",
    "privacy choices",
    "terms of service",
    "privacy policy",
]


class TestCanonicalize(unittest.TestCase):
    def test_lowercase_and_collapse(self):
        self.assertEqual(canonicalize("  Terms-of-Service. "), "terms of service")

    def test_empty(self):
        self.assertEqual(canonicalize(""), "")
        self.assertEqual(canonicalize(None), "")


class TestIsExcluded(unittest.TestCase):
    def test_exact_match(self):
        self.assertTrue(is_excluded("Terms of Service", PHRASES))

    def test_sub_phrase_match(self):
        # "privacy choices" matches the longer "Your Privacy Choices".
        self.assertTrue(is_excluded("Your Privacy Choices", PHRASES))

    def test_case_and_punctuation_insensitive(self):
        self.assertTrue(is_excluded("SKIP-TO-CONTENT", PHRASES))

    def test_no_match(self):
        self.assertFalse(is_excluded("Federal Reserve holds rates steady", PHRASES))

    def test_no_phrases(self):
        self.assertFalse(is_excluded("anything here now", []))
        self.assertFalse(is_excluded("anything here now", None))


class TestGetExcludedPhrases(unittest.TestCase):
    def test_loads_skipping_comments_and_blanks(self):
        path = None
        try:
            with tempfile.NamedTemporaryFile("w", suffix=".txt", delete=False) as f:
                f.write("# a comment\n\nskip to content\n\nprivacy choices\n# another\n")
                path = f.name
            self.assertEqual(get_excluded_phrases(path), ["skip to content", "privacy choices"])
        finally:
            if path:
                os.unlink(path)


class TestIsHeadlineExcluded(unittest.TestCase):
    def setUp(self):
        self.stopwords = {"the", "a", "an", "in", "on", "and", "of", "to"}

    def test_boilerplate_rejected(self):
        for text in ["Your Privacy Choices", "Skip to Content", "Terms of Service"]:
            with self.subTest(text=text):
                self.assertFalse(is_headline(text, self.stopwords, PHRASES))

    def test_real_headline_kept(self):
        self.assertTrue(is_headline("Senate passes major infrastructure bill", self.stopwords, PHRASES))


class TestPrepareHeadlinesExcluded(unittest.TestCase):
    def setUp(self):
        self.stopwords = {"the", "a", "an", "in", "on", "and", "of", "to"}

    @patch("services.headlines.requests.get")
    def test_boilerplate_links_are_filtered_out(self, mock_get):
        html = (
            "<html><body>"
            "<a href='/skip'>Skip to Content</a>"
            "<a href='/terms'>Terms of Service</a>"
            "<a href='/privacy'>Your Privacy Choices</a>"
            "<a href='/article'>Senate passes major infrastructure bill</a>"
            "</body></html>"
        )
        mock_response = MagicMock()
        mock_response.status_code = 200
        mock_response.content = html.encode("utf-8")
        mock_response.text = html
        mock_get.return_value = mock_response

        headlines = prepare_headlines(["https://example.com/news"], self.stopwords, excluded_phrases=PHRASES)

        texts = [h.display_text for h in headlines]
        self.assertEqual(texts, ["Senate passes major infrastructure bill"])


if __name__ == "__main__":
    unittest.main()