diff --git a/anya-updates.bundle b/anya-updates.bundle new file mode 100644 index 0000000..742eb91 Binary files /dev/null and b/anya-updates.bundle differ diff --git a/services/headlines.py b/services/headlines.py index ade23a7..5dd24a3 100644 --- a/services/headlines.py +++ b/services/headlines.py @@ -12,6 +12,33 @@ logger = logging.getLogger(__name__) DEFAULT_TIMEOUT = 10 MIN_HEADLINE_WORDS = 3 +# Common browser-like request headers. News sites frequently reject requests +# that look like minimal bots, so these make Anya look like a regular browser. +# Accept-Encoding is intentionally omitted so requests/urllib3 negotiates and +# decompresses a response it can actually handle (avoids brotli-only responses +# arriving as undecodable bytes). +DEFAULT_HEADERS = { + 'User-Agent': ( + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/124.0.0.0 Safari/537.36' + ), + 'Accept': ( + 'text/html,application/xhtml+xml,application/xml;q=0.9,' + 'image/avif,image/webp,image/apng,*/*;q=0.8' + ), + 'Accept-Language': 'en-US,en;q=0.9', + 'Cache-Control': 'max-age=0', + 'Upgrade-Insecure-Requests': '1', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-User': '?1', + 'sec-ch-ua': '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Windows"', +} + # Single streaming pass over the HTML: captures