anya/main.py

import argparse
import logging
from datetime import date

from services.dates import filter_by_date_window, parse_date_arg
from services.headlines import prepare_headlines
from services.normalization import get_aliases, get_excluded_phrases, get_stopwords
from services.sources import get_sources
from services.stories import cluster_stories

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s [%(levelname)s] [%(name)s]: %(message)s'
)
logger = logging.getLogger(__name__)

SOURCE_FILE = './resources/sources.txt'
STOPWORDS_FILE = './resources/stopwords.txt'
EXCLUDED_PHRASES_FILE = './resources/excluded_phrases.txt'
ALIASES_FILE = './resources/aliases.toml'
SIMILARITY_THRESHOLD = 0.64
MIN_SOURCES = 3

# Used so stories without a parsable date sort behind dated ones.
_MIN_DATE = date.min


def _describe_window(since, until):
    if since and until:
        return " (published %s to %s)" % (since, until)
    if since:
        return " (published on or after %s)" % since
    if until:
        return " (published on or before %s)" % until
    return ""


def parse_args():
    parser = argparse.ArgumentParser(
        description="Cluster headlines across news sources and print stories "
                    "covered by a minimum number of distinct outlets."
    )
    parser.add_argument('--min-sources', type=int, default=MIN_SOURCES, metavar='N',
                        help="Only output stories reported by at least N distinct sources "
                             "(default: %(default)s).")
    parser.add_argument('--threshold', type=float, default=SIMILARITY_THRESHOLD, metavar='T',
                        help="Cosine similarity used to consider two headlines the same story "
                             "(default: %(default)s).")
    parser.add_argument('--since', type=parse_date_arg, metavar='YYYY-MM-DD',
                        help="Only consider headlines published on or after this date.")
    parser.add_argument('--until', type=parse_date_arg, metavar='YYYY-MM-DD',
                        help="Only consider headlines published on or before this date.")
    parser.add_argument('--verbose', action='store_true',
                        help='Enable debug logging.')
    return parser.parse_args()


def main():
    args = parse_args()
    if args.verbose:
        logging.getLogger().setLevel(logging.DEBUG)

    if args.min_sources < 1:
        logger.error("--min-sources must be at least 1 (got %d).", args.min_sources)
        return

    if args.since and args.until and args.since > args.until:
        logger.error("--since (%s) is after --until (%s).", args.since, args.until)
        return

    logger.info("Starting duplicate headline detection application")
    logger.debug("Source file configured at: '%s'", SOURCE_FILE)
    logger.debug("Stopwords file configured at: '%s'", STOPWORDS_FILE)

    try:
        logger.info("Loading news sources from '%s'", SOURCE_FILE)
        sources = get_sources(SOURCE_FILE)
        logger.info("Loaded %d sources successfully", len(sources) if sources else 0)
    except Exception as e:
        logger.critical("Failed to load sources from '%s': %s", SOURCE_FILE, e, exc_info=True)
        return

    try:
        logger.info("Loading stopwords from '%s'", STOPWORDS_FILE)
        stopwords = get_stopwords(STOPWORDS_FILE)
        logger.info("Loaded %d stopwords successfully", len(stopwords) if stopwords else 0)
    except Exception as e:
        logger.critical("Failed to load stopwords from '%s': %s", STOPWORDS_FILE, e, exc_info=True)
        return

    try:
        logger.info("Loading excluded phrases from '%s'", EXCLUDED_PHRASES_FILE)
        excluded_phrases = get_excluded_phrases(EXCLUDED_PHRASES_FILE)
        logger.info("Loaded %d excluded phrases successfully", len(excluded_phrases))
    except Exception as e:
        logger.critical("Failed to load excluded phrases from '%s': %s", EXCLUDED_PHRASES_FILE, e, exc_info=True)
        return

    try:
        logger.info("Loading aliases from '%s'", ALIASES_FILE)
        aliases = get_aliases(ALIASES_FILE)
        logger.info("Loaded %d aliases successfully", len(aliases))
    except Exception as e:
        logger.critical("Failed to load aliases from '%s': %s", ALIASES_FILE, e, exc_info=True)
        return

    try:
        logger.info("Fetching and preparing headlines from %d sources", len(sources))
        headlines = prepare_headlines(sources, stopwords, excluded_phrases=excluded_phrases, aliases=aliases)
        logger.info("Total prepared headlines available for comparison: %d", len(headlines))
    except Exception as e:
        logger.critical("Failed during headline preparation: %s", e, exc_info=True)
        return

    if args.since is not None or args.until is not None:
        headlines, dropped_undated = filter_by_date_window(headlines, args.since, args.until)
        logger.info("Date window applied: %d headlines remain (%d dropped with no parseable date).",
                    len(headlines), dropped_undated)

    logger.info("Clustering headlines into stories (similarity threshold: %.2f)", args.threshold)
    stories = cluster_stories(headlines, args.threshold)

    qualifying = [s for s in stories if s.source_count >= args.min_sources]
    qualifying.sort(key=lambda s: (s.source_count, s.latest_date or _MIN_DATE), reverse=True)

    print()
    window = _describe_window(args.since, args.until)
    print("News stories covered by at least %d distinct sources" % args.min_sources + window)
    print("=" * 60)
    if not qualifying:
        print("No stories met the minimum number of sources.")
    else:
        for story in qualifying:
            date_label = story.latest_date.isoformat() if story.latest_date else "no date"
            print(f"\n[{story.source_count} source(s)] {story.representative} ({date_label})")
            print("    " + ", ".join(story.sources))

    logger.info(
        "Story clustering complete: %d total stories, %d with >= %d distinct sources.",
        len(stories), len(qualifying), args.min_sources,
    )


if __name__ == '__main__':
    main()