anya/services/normalization.py

import logging
import string

logger = logging.getLogger(__name__)


def get_stopwords(path):
    logger.info("Attempting to load stopwords from file: '%s'", path)
    try:
        with open(path, 'r', encoding='utf-8-sig') as stopwords_file:
            logger.debug("Opened stopwords file '%s'", path)
            lines = stopwords_file.read().splitlines()
            stopwords = set(lines)
            logger.info("Successfully loaded %d unique stopwords from '%s' (%d total lines)", len(stopwords), path, len(lines))
            return stopwords
    except FileNotFoundError:
        logger.error("Stopwords file not found at path: '%s'", path, exc_info=True)
        raise
    except PermissionError:
        logger.error("Permission denied accessing stopwords file: '%s'", path, exc_info=True)
        raise
    except Exception as e:
        logger.error("Failed to load stopwords from '%s': %s", path, e, exc_info=True)
        raise


def remove_stopwords(text, stopwords):
    logger.debug("Removing stopwords from text (%d chars): %r (available stopwords: %d)", len(text), text, len(stopwords))
    # split() without arguments handles all whitespace (spaces, tabs, newlines)
    words = text.split()
    sentence_words = []

    for word in words:
        # Strip surrounding punctuation and lowercase for comparison
        cleaned_word = word.strip(string.punctuation).lower()
        if cleaned_word and cleaned_word not in stopwords:
            sentence_words.append(word)
        elif cleaned_word in stopwords:
            logger.debug("Word filtered out as stopword: %r (original: %r)", cleaned_word, word)
        else:
            logger.debug("Word dropped (empty after stripping punctuation): %r", word)

    logger.debug("Filtered words: input=%d words, output=%d words -> %s", len(words), len(sentence_words), sentence_words)
    return sentence_words


def normalize_headline(text, stopwords):
    logger.debug("Starting normalization of headline: %r", text)
    headline = text.strip().lower()
    punctuation = string.punctuation
    for char in punctuation:
        headline = headline.replace(char, '')
    logger.debug("Headline after lowercasing and punctuation removal: %r", headline)
    normalized = remove_stopwords(headline, stopwords)
    logger.debug("Completed normalization for %r -> %s", text, normalized)
    return normalized