Add recap fields and scraping functionality for game details

src/models/game.py

@@ -18,6 +18,10 @@ class Game:
         - `box_score`       The scoring summary of the game (optional)
         - `score_breakdown` The scoring breakdown of the game (optional)
         - 'ticket_link'    The ticket link for the game (optional)
+        - `recap_link`     The recap article link (optional)
+        - `recap_article_title` The recap article title (optional)
+        - `recap_article_image` The recap article image (optional)
+        - `recap_published_at` The recap article publication date (optional)
     """
 
     def __init__(
@@ -37,6 +41,10 @@ def __init__(
         team=None,
         utc_date=None,
         ticket_link=None,
+        recap_link=None,
+        recap_article_title=None,
+        recap_article_image=None,
+        recap_published_at=None,
     ):
         self.id = id if id else str(ObjectId())
         self.city = city
@@ -53,6 +61,10 @@ def __init__(
         self.team = team
         self.utc_date = utc_date
         self.ticket_link = ticket_link
+        self.recap_link = recap_link
+        self.recap_article_title = recap_article_title
+        self.recap_article_image = recap_article_image
+        self.recap_published_at = recap_published_at
 
     def to_dict(self):
         """
@@ -74,6 +86,10 @@ def to_dict(self):
             "team": self.team,
             "utc_date": self.utc_date,
             "ticket_link": self.ticket_link,
+            "recap_link": self.recap_link,
+            "recap_article_title": self.recap_article_title,
+            "recap_article_image": self.recap_article_image,
+            "recap_published_at": self.recap_published_at,
         }
 
     @staticmethod
@@ -97,4 +113,8 @@ def from_dict(data) -> None:
             team=data.get("team"),
             utc_date=data.get("utc_date"),
             ticket_link=data.get("ticket_link"),
+            recap_link=data.get("recap_link"),
+            recap_article_title=data.get("recap_article_title"),
+            recap_article_image=data.get("recap_article_image"),
+            recap_published_at=data.get("recap_published_at"),
         )

src/mutations/create_game.py

@@ -13,11 +13,15 @@ class Arguments:
         result = String(required=False)
         sport = String(required=True)
         state = String(required=True)
-        time = String(required=True)
+        time = String(required=False)
         box_score = String(required=False)
         score_breakdown = String(required=False)
         utc_date = String(required=False)
         ticket_link = String(required=False)
+        recap_link = String(required=False)
+        recap_article_title = String(required=False)
+        recap_article_image = String(required=False)
+        recap_published_at = String(required=False)
 
     game = Field(lambda: GameType)
 
@@ -36,7 +40,11 @@ def mutate(
         box_score=None,
         score_breakdown=None,
         utc_date=None,
-        ticket_link=None
+        ticket_link=None,
+        recap_link=None,
+        recap_article_title=None,
+        recap_article_image=None,
+        recap_published_at=None,
     ):
         game_data = {
             "city": city,
@@ -51,7 +59,11 @@ def mutate(
             "box_score": box_score,
             "score_breakdown": score_breakdown,
             "utc_date": utc_date,
-            "ticket_link": ticket_link
+            "ticket_link": ticket_link,
+            "recap_link": recap_link,
+            "recap_article_title": recap_article_title,
+            "recap_article_image": recap_article_image,
+            "recap_published_at": recap_published_at,
         }
         new_game = GameService.create_game(game_data)
-        return CreateGame(game=new_game)
+        return CreateGame(game=new_game)

src/repositories/game_repository.py

@@ -180,6 +180,35 @@ def find_by_tournament_key_fields(city, date, gender, location, sport, state):
 
         return [Game.from_dict(game) for game in games]
 
+    @staticmethod
+    def find_by_scraper_match_levels(date, sport, gender, opponent_id, city, state, location):
+        """Find a game without matching unrelated opponents by date alone."""
+        game_collection = db["game"]
+        base_query = {"date": date, "sport": sport, "gender": gender}
+        queries = [
+            {
+                **base_query,
+                "opponent_id": opponent_id,
+                "city": city,
+                "state": state,
+                "location": location,
+            },
+            {**base_query, "opponent_id": opponent_id},
+        ]
+
+        for level, query in enumerate(queries, start=1):
+            candidates = list(game_collection.find(query))
+            if len(candidates) == 1:
+                return Game.from_dict(candidates[0]), level
+            if len(candidates) > 1:
+                logger.warning(
+                    "Multiple games matched at match level %s; skipping",
+                    level,
+                )
+                return None, level
+
+        return None, None
+
     @staticmethod
     def find_by_sport(sport):
         """
@@ -221,7 +250,9 @@ def find_games_by_sport_gender_after_date(sport, gender, after_date=None):
         }
 
         if after_date:
-            query["utc_date"] = {"$gt": after_date}
+            query["utc_date"] = {
+                "$gt": after_date.isoformat() if hasattr(after_date, "isoformat") else after_date
+            }
 
         games = game_collection.find(query)
         return [Game.from_dict(game) for game in games]

src/scrapers/game_details_scrape.py

@@ -1,7 +1,13 @@
 import re
+import logging
 import requests
 from bs4 import BeautifulSoup
+from urllib.parse import urljoin
 from src.utils.constants import *
+from src.utils.helpers import is_allowed_url
+
+
+logger = logging.getLogger(__name__)
 
 def clean_name(name):
     """Strip extra information from player names, keeping only first and last name."""
@@ -22,9 +28,103 @@ def clean_name(name):
     return cleaned
 
 def fetch_page(url):
-    response = requests.get(url)
+    if not is_allowed_url(url):
+        raise ValueError(f"Unapproved box score URL: {url}")
+    response = requests.get(url, headers=HTTP_REQUEST_HEADERS, timeout=30)
+    response.raise_for_status()
     return BeautifulSoup(response.text, 'html.parser')
 
+
+def fetch_recap_page(url):
+    if not is_allowed_url(url):
+        raise ValueError(f"Unapproved recap URL: {url}")
+    response = requests.get(url, headers=HTTP_REQUEST_HEADERS, timeout=30)
+    response.raise_for_status()
+    return BeautifulSoup(response.text, "html.parser")
+
+
+def _metadata_content(soup, selector):
+    tag = soup.select_one(selector)
+    return tag.get("content") if tag else None
+
+
+def _tag_text_or_datetime(tag):
+    if not tag:
+        return None
+    return tag.get("datetime") or tag.get_text(" ", strip=True) or None
+
+
+def _published_at(soup):
+    """Extract the article's publication timestamp without empty/related dates."""
+    # Older Sidearm stories put the publication timestamp in this story-date
+    # block without a pubdate attribute.
+    for tag in soup.select(SIDEARM_STORY_PUBLISHED_TIME_FALLBACK):
+        value = _tag_text_or_datetime(tag)
+        if value:
+            return value
+
+    metadata_value = _metadata_content(soup, 'meta[property="article:published_time"]')
+    if metadata_value:
+        return metadata_value
+
+    # Newer stories use pubdate. Ignore empty placeholders and compact dates
+    # from related-story cards (for example, "09.11.26").
+    for tag in soup.select(SIDEARM_STORY_PUBLISHED_TIME):
+        value = _tag_text_or_datetime(tag)
+        if value and re.search(r"\b20\d{2}\b", value):
+            return value
+
+    return None
+
+
+def _first_image_url(soup, base_url):
+    image = soup.select_one(SIDEARM_STORY_IMAGE)
+    if image:
+        image_url = image.get("data-src") or image.get("src")
+        if image_url:
+            return urljoin(base_url, image_url)
+
+    # Some Sidearm stories put the responsive image only in a <source> tag.
+    source = soup.select_one(".sidearm-story-template-media source")
+    if source:
+        srcset = source.get("srcset", "").split(",")[0].strip().split(" ")[0]
+        if srcset:
+            return urljoin(base_url, srcset)
+
+    metadata_image = _metadata_content(soup, 'meta[property="og:image"]')
+    return urljoin(base_url, metadata_image) if metadata_image else None
+
+
+def scrape_sidearm_story_recap(url):
+    """Scrape the article metadata from a Sidearm recap page.
+
+    ``None`` means the page could not be fetched or parsed. A dictionary with
+    nullable fields means the page was fetched successfully, which lets the
+    schedule scraper preserve existing article data on transient failures.
+    """
+    if not url:
+        return None
+
+    try:
+        soup = fetch_recap_page(url)
+    except (requests.RequestException, ValueError) as exc:
+        logger.warning("Unable to fetch recap page %s: %s", url, exc)
+        return None
+    except Exception as exc:
+        logger.exception("Unexpected error fetching recap page %s: %s", url, exc)
+        return None
+
+    headline = soup.select_one(SIDEARM_STORY_HEADLINE)
+    return {
+        "recap_article_title": (
+            headline.get_text(" ", strip=True)
+            if headline
+            else _metadata_content(soup, 'meta[property="og:title"]')
+        ),
+        "recap_article_image": _first_image_url(soup, url),
+        "recap_published_at": _published_at(soup),
+    }
+
 def extract_teams_and_scores(box_score_section, sport):
     score_table = box_score_section.find(TAG_TABLE, class_=CLASS_SIDEARM_TABLE)
     team_names = []
@@ -79,8 +179,6 @@ def soccer_summary(box_score_section):
                     'cor_score': cornell_score,
                     'opp_score': opp_score
                 })
-    if not summary:
-        summary = [{"message": "No scoring events in this game."}]
     return summary
 
 def football_summary(box_score_section):
@@ -105,8 +203,6 @@ def football_summary(box_score_section):
                     'cor_score': cornell_score,
                     'opp_score': opp_score
                 })
-    if not summary:
-        summary = [{"message": "No scoring events in this game."}]
     return summary
 
 def hockey_summary(box_score_section):
@@ -139,8 +235,6 @@ def hockey_summary(box_score_section):
                     'opp_score': opp_score,
                     'description': f"Scored by {scorer}. Assisted by {assist}."
                 })
-    if not summary:
-        summary = [{"message": "No scoring events in this game."}]
     return summary
 
 def field_hockey_summary(box_score_section):
@@ -167,8 +261,6 @@ def field_hockey_summary(box_score_section):
                 'cor_score': cornell_score,
                 'opp_score': opp_score
             })
-    if not summary:
-        summary = [{"message": "No scoring events in this game."}]
     return summary
 
 def lacrosse_summary(box_score_section):
@@ -201,8 +293,6 @@ def lacrosse_summary(box_score_section):
                     'cor_score': cor_score,
                     'opp_score': opp_score,
                 })
-    if not summary:
-        summary = [{"message": "No scoring events in this game."}]
     return summary
 
 def baseball_summary(box_score_section):
@@ -225,6 +315,30 @@ def baseball_summary(box_score_section):
                     'cor_score': cor_score,
                     'opp_score': opp_score
                 })
+    return summary
+
+def softball_summary(box_score_section):
+    summary = []
+    scoring_section = box_score_section.find(TAG_SECTION, {ATTR_ARIA_LABEL: LABEL_SCORING_SUMMARY})
+    if scoring_section:
+        scoring_rows = scoring_section.find(TAG_TBODY)
+        if scoring_rows:
+            for row in scoring_rows.find_all(TAG_TR):
+                cells = row.find_all(TAG_TD)
+                team = cells[0].find(TAG_IMG)[ATTR_ALT]
+                inning = cells[3].get_text(strip=True)
+                description = cells[4]
+                span = description.find(TAG_SPAN)
+                if span:
+                    span.extract()
+                summary.append({
+                    'team': team,
+                    'period': inning,
+                    'inning': inning,
+                    'description': description.get_text(strip=True),
+                    'cor_score': int(cells[5].get_text(strip=True) or 0),
+                    'opp_score': int(cells[6].get_text(strip=True) or 0),
+                })
     if not summary:
         summary = [{"message": "No scoring events in this game."}]
     return summary
@@ -272,6 +386,7 @@ def scrape_game(url, sport):
         'field hockey': (lambda: extract_teams_and_scores(box_score_section, 'field hockey'), field_hockey_summary),
         'lacrosse': (lambda: extract_teams_and_scores(box_score_section, 'lacrosse'), lacrosse_summary),
         'baseball': (lambda: extract_teams_and_scores(box_score_section, 'baseball'), baseball_summary),
+        'softball': (lambda: extract_teams_and_scores(box_score_section, 'softball'), softball_summary),
         'basketball': (lambda: extract_teams_and_scores(box_score_section, 'basketball'), lambda _: []),
     }
 
@@ -288,7 +403,7 @@ def scrape_game(url, sport):
         return {
             'teams': team_names,
             'scores': scores,
-            'scoring_summary': scoring_summary or [{"message": "No scoring events in this game."}]
+            'scoring_summary': scoring_summary
         }
 
-    return {"error": "Sport parser not found"}
+    return {"error": "Sport parser not found"}