anya/main.py

import logging
from services.headlines import prepare_headlines
from services.normalization import get_stopwords, normalize_headline
from services.sources import get_sources

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s [%(levelname)s] [%(name)s]: %(message)s'
)
logger = logging.getLogger(__name__)

SOURCE_FILE = './resources/sources.txt'
STOPWORDS_FILE = './resources/stopwords.txt'
SIMILARITY_THRESHOLD = 0.75


def main():
    logger.info("Starting duplicate headline detection application")
    logger.debug("Source file configured at: '%s'", SOURCE_FILE)
    logger.debug("Stopwords file configured at: '%s'", STOPWORDS_FILE)

    try:
        logger.info("Loading news sources from '%s'", SOURCE_FILE)
        sources = get_sources(SOURCE_FILE)
        logger.info("Loaded %d sources successfully", len(sources) if sources else 0)
    except Exception as e:
        logger.critical("Failed to load sources from '%s': %s", SOURCE_FILE, e, exc_info=True)
        return

    try:
        logger.info("Loading stopwords from '%s'", STOPWORDS_FILE)
        stopwords = get_stopwords(STOPWORDS_FILE)
        logger.info("Loaded %d stopwords successfully", len(stopwords) if stopwords else 0)
    except Exception as e:
        logger.critical("Failed to load stopwords from '%s': %s", STOPWORDS_FILE, e, exc_info=True)
        return

    try:
        logger.info("Fetching and preparing headlines from %d sources", len(sources))
        headlines = prepare_headlines(sources, stopwords)
        logger.info("Total prepared headlines available for comparison: %d", len(headlines))
    except Exception as e:
        logger.critical("Failed during headline preparation: %s", e, exc_info=True)
        return

    total_comparisons = (len(headlines) * (len(headlines) - 1)) // 2 if len(headlines) > 1 else 0
    logger.info("Beginning pairwise headline comparisons (total comparisons to execute: %d)", total_comparisons)

    duplicate_count = 0
    comparison_idx = 0
    for i in range(len(headlines)):
        for j in range(i + 1, len(headlines)):
            comparison_idx += 1
            logger.debug("Comparison [%d/%d]: Headline %d vs Headline %d", comparison_idx, total_comparisons, i, j)
            try:
                similarity_score = headlines[i].compare_headlines(headlines[j])
                logger.debug("Similarity score between [%d] and [%d] is %.4f (threshold: %f)", i, j, similarity_score, SIMILARITY_THRESHOLD)
                if similarity_score >= SIMILARITY_THRESHOLD:
                    duplicate_count += 1
                    logger.warning("Duplicate/similar headline match found (score: %.4f < %f): '%s' vs '%s'",
                                   similarity_score, SIMILARITY_THRESHOLD, headlines[i].display_text, headlines[j].display_text)
                    print(f"Duplicate headlines found: {headlines[i].display_text}")
            except Exception as e:
                logger.error("Error during comparison between headline %d (%r) and headline %d (%r): %s",
                             i, headlines[i].display_text, j, headlines[j].display_text, e, exc_info=True)

    logger.info("Headline comparison completed. Evaluated %d pairs and found %d duplicate alerts.",
                comparison_idx, duplicate_count)


if __name__ == '__main__':
    main()