diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..029519e
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+__pycache__/
+*.py[cod]
\ No newline at end of file
diff --git a/__pycache__/main.cpython-314.pyc b/__pycache__/main.cpython-314.pyc
deleted file mode 100644
index 346100e..0000000
Binary files a/__pycache__/main.cpython-314.pyc and /dev/null differ
diff --git a/main.py b/main.py
index d788e5a..060d4c8 100644
--- a/main.py
+++ b/main.py
@@ -1,7 +1,12 @@
+import argparse
import logging
+from datetime import date
+
+from services.dates import filter_by_date_window, parse_date_arg
from services.headlines import prepare_headlines
-from services.normalization import get_stopwords, normalize_headline
+from services.normalization import get_stopwords
from services.sources import get_sources
+from services.stories import cluster_stories
logging.basicConfig(
level=logging.INFO,
@@ -12,9 +17,55 @@ logger = logging.getLogger(__name__)
SOURCE_FILE = './resources/sources.txt'
STOPWORDS_FILE = './resources/stopwords.txt'
SIMILARITY_THRESHOLD = 0.75
+MIN_SOURCES = 2
+
+# Used so stories without a parsable date sort behind dated ones.
+_MIN_DATE = date.min
+
+
+def _describe_window(since, until):
+ if since and until:
+ return " (published %s to %s)" % (since, until)
+ if since:
+ return " (published on or after %s)" % since
+ if until:
+ return " (published on or before %s)" % until
+ return ""
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(
+ description="Cluster headlines across news sources and print stories "
+ "covered by a minimum number of distinct outlets."
+ )
+ parser.add_argument('--min-sources', type=int, default=MIN_SOURCES, metavar='N',
+ help="Only output stories reported by at least N distinct sources "
+ "(default: %(default)s).")
+ parser.add_argument('--threshold', type=float, default=SIMILARITY_THRESHOLD, metavar='T',
+ help="Cosine similarity used to consider two headlines the same story "
+ "(default: %(default)s).")
+ parser.add_argument('--since', type=parse_date_arg, metavar='YYYY-MM-DD',
+ help="Only consider headlines published on or after this date.")
+ parser.add_argument('--until', type=parse_date_arg, metavar='YYYY-MM-DD',
+ help="Only consider headlines published on or before this date.")
+ parser.add_argument('--verbose', action='store_true',
+ help='Enable debug logging.')
+ return parser.parse_args()
def main():
+ args = parse_args()
+ if args.verbose:
+ logging.getLogger().setLevel(logging.DEBUG)
+
+ if args.min_sources < 1:
+ logger.error("--min-sources must be at least 1 (got %d).", args.min_sources)
+ return
+
+ if args.since and args.until and args.since > args.until:
+ logger.error("--since (%s) is after --until (%s).", args.since, args.until)
+ return
+
logger.info("Starting duplicate headline detection application")
logger.debug("Source file configured at: '%s'", SOURCE_FILE)
logger.debug("Stopwords file configured at: '%s'", STOPWORDS_FILE)
@@ -43,30 +94,34 @@ def main():
logger.critical("Failed during headline preparation: %s", e, exc_info=True)
return
- total_comparisons = (len(headlines) * (len(headlines) - 1)) // 2 if len(headlines) > 1 else 0
- logger.info("Beginning pairwise headline comparisons (total comparisons to execute: %d)", total_comparisons)
-
- duplicate_count = 0
- comparison_idx = 0
- for i in range(len(headlines)):
- for j in range(i + 1, len(headlines)):
- comparison_idx += 1
- logger.debug("Comparison [%d/%d]: Headline %d vs Headline %d", comparison_idx, total_comparisons, i, j)
- try:
- similarity_score = headlines[i].compare_headlines(headlines[j])
- logger.debug("Similarity score between [%d] and [%d] is %.4f (threshold: %f)", i, j, similarity_score, SIMILARITY_THRESHOLD)
- if similarity_score >= SIMILARITY_THRESHOLD:
- duplicate_count += 1
- logger.warning("Duplicate/similar headline match found (score: %.4f < %f): '%s' vs '%s'",
- similarity_score, SIMILARITY_THRESHOLD, headlines[i].display_text, headlines[j].display_text)
- print(f"Duplicate headlines found: {headlines[i].display_text}")
- except Exception as e:
- logger.error("Error during comparison between headline %d (%r) and headline %d (%r): %s",
- i, headlines[i].display_text, j, headlines[j].display_text, e, exc_info=True)
-
- logger.info("Headline comparison completed. Evaluated %d pairs and found %d duplicate alerts.",
- comparison_idx, duplicate_count)
+ if args.since is not None or args.until is not None:
+ headlines, dropped_undated = filter_by_date_window(headlines, args.since, args.until)
+ logger.info("Date window applied: %d headlines remain (%d dropped with no parseable date).",
+ len(headlines), dropped_undated)
+
+ logger.info("Clustering headlines into stories (similarity threshold: %.2f)", args.threshold)
+ stories = cluster_stories(headlines, args.threshold)
+
+ qualifying = [s for s in stories if s.source_count >= args.min_sources]
+ qualifying.sort(key=lambda s: (s.source_count, s.latest_date or _MIN_DATE), reverse=True)
+
+ print()
+ window = _describe_window(args.since, args.until)
+ print("News stories covered by at least %d distinct sources" % args.min_sources + window)
+ print("=" * 60)
+ if not qualifying:
+ print("No stories met the minimum number of sources.")
+ else:
+ for story in qualifying:
+ date_label = story.latest_date.isoformat() if story.latest_date else "no date"
+ print(f"\n[{story.source_count} source(s)] {story.representative} ({date_label})")
+ print(" " + ", ".join(story.sources))
+
+ logger.info(
+ "Story clustering complete: %d total stories, %d with >= %d distinct sources.",
+ len(stories), len(qualifying), args.min_sources,
+ )
if __name__ == '__main__':
- main()
+ main()
\ No newline at end of file
diff --git a/services/__pycache__/headlines.cpython-314.pyc b/services/__pycache__/headlines.cpython-314.pyc
deleted file mode 100644
index 3696f10..0000000
Binary files a/services/__pycache__/headlines.cpython-314.pyc and /dev/null differ
diff --git a/services/__pycache__/normalization.cpython-314.pyc b/services/__pycache__/normalization.cpython-314.pyc
deleted file mode 100644
index 2b53b51..0000000
Binary files a/services/__pycache__/normalization.cpython-314.pyc and /dev/null differ
diff --git a/services/__pycache__/similarity.cpython-314.pyc b/services/__pycache__/similarity.cpython-314.pyc
deleted file mode 100644
index 260ad84..0000000
Binary files a/services/__pycache__/similarity.cpython-314.pyc and /dev/null differ
diff --git a/services/__pycache__/sources.cpython-314.pyc b/services/__pycache__/sources.cpython-314.pyc
deleted file mode 100644
index 1752280..0000000
Binary files a/services/__pycache__/sources.cpython-314.pyc and /dev/null differ
diff --git a/services/dates.py b/services/dates.py
new file mode 100644
index 0000000..3a9dd19
--- /dev/null
+++ b/services/dates.py
@@ -0,0 +1,95 @@
+import logging
+from datetime import date, datetime, timezone
+from email.utils import parsedate_to_datetime
+
+logger = logging.getLogger(__name__)
+
+# Common absolute-date formats encountered in news markup when ISO parsing fails.
+_FALLBACK_FORMATS = (
+ "%Y-%m-%d %H:%M:%S",
+ "%Y-%m-%d %H:%M",
+ "%Y/%m/%d %H:%M:%S",
+ "%B %d, %Y",
+ "%b %d, %Y",
+ "%B %d, %Y %I:%M %p",
+ "%b %d, %Y %I:%M %p",
+)
+
+
+def parse_datetime(value):
+ """Best-effort parse of a publish timestamp into a ``datetime``.
+
+ Handles ISO 8601 (including trailing ``Z`` and offsets), RFC 2822/email
+ dates, and a few common human formats. Returns ``None`` when the value
+ cannot be parsed — relative strings like "3 hours ago" are intentionally
+ skipped.
+ """
+ if not value or not isinstance(value, str):
+ return None
+ text = value.strip()
+ if not text:
+ return None
+
+ try:
+ return datetime.fromisoformat(text)
+ except ValueError:
+ pass
+
+ try:
+ return parsedate_to_datetime(text)
+ except (TypeError, ValueError, OverflowError):
+ pass
+
+ for fmt in _FALLBACK_FORMATS:
+ try:
+ return datetime.strptime(text, fmt)
+ except ValueError:
+ pass
+
+ logger.debug("Could not parse datetime from value %r", value)
+ return None
+
+
+def parse_date_arg(value):
+ """Parse a ``YYYY-MM-DD`` CLI argument, raising a friendly error otherwise."""
+ try:
+ return date.fromisoformat(value)
+ except ValueError:
+ raise ValueError("Invalid date %r; expected YYYY-MM-DD" % value)
+
+
+def date_part(dt):
+ """Return ``dt`` as a ``date``, normalized to UTC when timezone-aware."""
+ if dt.tzinfo is not None:
+ return dt.astimezone(timezone.utc).date()
+ return dt.date()
+
+
+def filter_by_date_window(headlines, since=None, until=None):
+ """Keep headlines whose publication date falls in ``[since, until]``.
+
+ ``since``/``until`` are ``datetime.date`` (or ``None`` for an open bound).
+ When either bound is set, headlines with no parsable publication date are
+ dropped because their recency cannot be established; without any bound they
+ are all kept. Returns ``(kept, dropped_undated)``.
+ """
+ kept = []
+ dropped_undated = 0
+ bounding = since is not None or until is not None
+
+ for h in headlines:
+ if h.published_at is None:
+ if bounding:
+ dropped_undated += 1
+ continue
+ kept.append(h)
+ continue
+
+ d = date_part(h.published_at)
+ if since is not None and d < since:
+ continue
+ if until is not None and d > until:
+ continue
+ kept.append(h)
+
+ return kept, dropped_undated
\ No newline at end of file
diff --git a/services/headlines.py b/services/headlines.py
index 01614d4..ade23a7 100644
--- a/services/headlines.py
+++ b/services/headlines.py
@@ -1,6 +1,9 @@
import logging
-from re import findall
+import re
+
import requests
+
+from services.dates import parse_datetime
from services.normalization import normalize_headline
from structs.headline import Headline
@@ -9,6 +12,52 @@ logger = logging.getLogger(__name__)
DEFAULT_TIMEOUT = 10
MIN_HEADLINE_WORDS = 3
+# Single streaming pass over the HTML: captures " # group 1 attrs, group 2 text
+ r"|(?:(?:datetime|dateTime)\s*=\s*[\"']([^\"']+)[\"'])" # group 3 datetime attr
+ r"|<(?:article|li)\b[^>]*>" # container boundary (no group)
+ r"|<(?:a|span)\b[^>]*>\s*([^<]*?)\s*", # group 4 headline text
+ re.IGNORECASE | re.DOTALL,
+)
+
+
+def _extract_candidates(source_content):
+ """Yield ``(text, published_at)`` tuples in document order.
+
+ Each headline is associated with the most recent publication timestamp seen
+ before it. Timestamps reset at each ````/``
`` boundary so a
+ headline with no date of its own does not inherit another story's timestamp.
+ """
+ candidates = []
+ last_published = None
+ for match in _CANDIDATE_RE.finditer(source_content):
+ if match.group(1) is not None:
+ # A element: prefer an explicit datetime/title
+ # attribute, otherwise fall back to its inner text.
+ attrs = match.group(1)
+ attr_match = re.search(r'(?:datetime|dateTime)\s*=\s*[\"\']([^\"\']+)[\"\']', attrs, re.IGNORECASE) \
+ or re.search(r'title\s*=\s*[\"\']([^\"\']+)[\"\']', attrs, re.IGNORECASE)
+ raw = attr_match.group(1) if attr_match else match.group(2)
+ parsed = parse_datetime(raw)
+ if parsed is not None:
+ last_published = parsed
+ elif match.group(3) is not None:
+ # A datetime attribute on some non-