anya/services/headlines.py

import logging
from re import findall
import requests
from services.normalization import normalize_headline
from structs.headline import Headline

logger = logging.getLogger(__name__)

DEFAULT_TIMEOUT = 10
MIN_HEADLINE_WORDS = 3


def is_headline(text, stopwords=None):
    if not text or not isinstance(text, str):
        return False
    cleaned_text = text.strip()
    if not cleaned_text:
        return False
    words = cleaned_text.split()
    if len(words) < MIN_HEADLINE_WORDS:
        logger.debug("Text rejected as headline (fewer than %d words): %r", MIN_HEADLINE_WORDS, cleaned_text)
        return False
    if not any(c.isalnum() for c in cleaned_text):
        logger.debug("Text rejected as headline (no alphanumeric characters): %r", cleaned_text)
        return False
    if stopwords is not None:
        normalized = normalize_headline(cleaned_text, stopwords)
        if not normalized:
            logger.debug("Text rejected as headline (no meaningful tokens after stopword removal): %r", cleaned_text)
            return False
    return True


def prepare_headlines(sources, stopwords, timeout=DEFAULT_TIMEOUT):
    logger.info("Starting preparation of headlines for %d sources", len(sources) if sources else 0)
    headlines = []
    if not sources:
        logger.warning("No sources provided to prepare_headlines.")
        return headlines

    for idx, source in enumerate(sources, start=1):
        if not source or not source.strip():
            logger.warning("Skipping empty source at index %d", idx)
            continue

        source_url = source.strip()
        logger.info("Fetching source [%d/%d]: '%s'", idx, len(sources), source_url)
        try:
            response = requests.get(source_url, allow_redirects=True, timeout=timeout, headers={'User-Agent': 'Anya news bot'})
            logger.debug("Received HTTP response %d for '%s' (content length: %d bytes)",
                         response.status_code, source_url, len(response.content))
            if response.status_code != 200:
                logger.warning("Source '%s' returned non-200 status code: %d", source_url, response.status_code)
            source_content = response.text
        except requests.exceptions.Timeout as e:
            logger.error("Request timed out for source '%s': %s", source_url, e, exc_info=True)
            continue
        except requests.exceptions.RequestException as e:
            logger.error("HTTP request failed for source '%s': %s", source_url, e, exc_info=True)
            continue
        except Exception as e:
            logger.error("Unexpected error fetching source '%s': %s", source_url, e, exc_info=True)
            continue

        logger.debug("Parsing HTML content from '%s' for headline candidates", source_url)
        try:
            link_texts = findall(r'<(?:a|span)\b[^>]*>\s*([^<]*?)\s*</(?:a|span)>', source_content)
            logger.info("Found %d candidate tags in source '%s'", len(link_texts), source_url)
        except Exception as e:
            logger.error("Regex extraction failed on content from '%s': %s", source_url, e, exc_info=True)
            continue

        source_headlines_count = 0
        for tag_idx, link_text in enumerate(link_texts, start=1):
            cleaned_text = link_text.strip()
            if not cleaned_text:
                logger.debug("Skipping empty tag text at position %d from '%s'", tag_idx, source_url)
                continue
            if not is_headline(cleaned_text, stopwords):
                logger.debug("Skipping non-headline tag text at position %d from '%s': %r", tag_idx, source_url, cleaned_text)
                continue
            logger.debug("Processing tag [%d/%d] from '%s': %r", tag_idx, len(link_texts), source_url, cleaned_text)
            try:
                normalized_headline = normalize_headline(cleaned_text, stopwords)
                headline = Headline(cleaned_text, normalized_headline)
                headlines.append(headline)
                source_headlines_count += 1
            except Exception as e:
                logger.error("Failed to normalize/create headline for text %r from '%s': %s", cleaned_text, source_url, e, exc_info=True)

        logger.info("Successfully extracted %d headlines from source '%s'", source_headlines_count, source_url)

    logger.info("Finished preparing headlines. Total headlines collected across all sources: %d", len(headlines))
    return headlines