Add browser-like request headers for resilience

News sites often reject minimal-bot requests; send a full browser header
set (User-Agent, Accept, Sec-Fetch-*, client hints). Accept-Encoding is
left to requests/urllib3 to avoid undecodable brotli responses.

services/headlines.py

@@ -12,6 +12,33 @@ logger = logging.getLogger(__name__)
 DEFAULT_TIMEOUT = 10
 MIN_HEADLINE_WORDS = 3
 
+# Common browser-like request headers. News sites frequently reject requests
+# that look like minimal bots, so these make Anya look like a regular browser.
+# Accept-Encoding is intentionally omitted so requests/urllib3 negotiates and
+# decompresses a response it can actually handle (avoids brotli-only responses
+# arriving as undecodable bytes).
+DEFAULT_HEADERS = {
+    'User-Agent': (
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
+        'AppleWebKit/537.36 (KHTML, like Gecko) '
+        'Chrome/124.0.0.0 Safari/537.36'
+    ),
+    'Accept': (
+        'text/html,application/xhtml+xml,application/xml;q=0.9,'
+        'image/avif,image/webp,image/apng,*/*;q=0.8'
+    ),
+    'Accept-Language': 'en-US,en;q=0.9',
+    'Cache-Control': 'max-age=0',
+    'Upgrade-Insecure-Requests': '1',
+    'Sec-Fetch-Dest': 'document',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+    'Sec-Fetch-User': '?1',
+    'sec-ch-ua': '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
+    'sec-ch-ua-mobile': '?0',
+    'sec-ch-ua-platform': '"Windows"',
+}
+
 # Single streaming pass over the HTML: captures <time> markers (with an optional
 # datetime/title attribute or inner text) and <a>/<span> headline text, so every
 # headline can be associated with the most recent publication timestamp seen
@@ -80,7 +107,8 @@ def is_headline(text, stopwords=None):
     return True
 
 
-def prepare_headlines(sources, stopwords, timeout=DEFAULT_TIMEOUT):
+def prepare_headlines(sources, stopwords, timeout=DEFAULT_TIMEOUT, headers=None):
+    request_headers = headers if headers is not None else DEFAULT_HEADERS
     logger.info("Starting preparation of headlines for %d sources", len(sources) if sources else 0)
     headlines = []
     if not sources:
@@ -95,7 +123,7 @@ def prepare_headlines(sources, stopwords, timeout=DEFAULT_TIMEOUT):
         source_url = source.strip()
         logger.info("Fetching source [%d/%d]: '%s'", idx, len(sources), source_url)
         try:
-            response = requests.get(source_url, allow_redirects=True, timeout=timeout, headers={'User-Agent': 'Anya news bot'})
+            response = requests.get(source_url, allow_redirects=True, timeout=timeout, headers=request_headers)
             logger.debug("Received HTTP response %d for '%s' (content length: %d bytes)",
                          response.status_code, source_url, len(response.content))
             if response.status_code != 200:

tests/test_headlines.py

@@ -1,7 +1,7 @@
 import unittest
 from unittest.mock import patch, MagicMock
 import requests
-from services.headlines import prepare_headlines, is_headline, DEFAULT_TIMEOUT
+from services.headlines import prepare_headlines, is_headline, DEFAULT_TIMEOUT, DEFAULT_HEADERS
 
 
 class TestHeadlinesTimeout(unittest.TestCase):
@@ -19,7 +19,7 @@ class TestHeadlinesTimeout(unittest.TestCase):
         sources = ["https://example.com/news"]
         headlines = prepare_headlines(sources, self.stopwords)
 
-        mock_get.assert_called_once_with("https://example.com/news", allow_redirects=True, timeout=DEFAULT_TIMEOUT, headers={'User-Agent': 'Anya news bot'})
+        mock_get.assert_called_once_with("https://example.com/news", allow_redirects=True, timeout=DEFAULT_TIMEOUT, headers=DEFAULT_HEADERS)
         self.assertEqual(len(headlines), 1)
         self.assertEqual(headlines[0].display_text, "Default Timeout Headline")
 
@@ -34,7 +34,7 @@ class TestHeadlinesTimeout(unittest.TestCase):
         sources = ["https://example.com/news"]
         headlines = prepare_headlines(sources, self.stopwords, timeout=10)
 
-        mock_get.assert_called_once_with("https://example.com/news", allow_redirects=True, timeout=10, headers={'User-Agent': 'Anya news bot'})
+        mock_get.assert_called_once_with("https://example.com/news", allow_redirects=True, timeout=10, headers=DEFAULT_HEADERS)
         self.assertEqual(len(headlines), 1)
 
     @patch("services.headlines.requests.get")