From a44280fcebf7494f036b63fe5d313ab1308e933b Mon Sep 17 00:00:00 2001 From: claiireyu Date: Sat, 12 Sep 2026 17:13:17 -0400 Subject: [PATCH 1/4] Add recap fields and scraping functionality for game details - Extend Game model to include recap article fields. - Update CreateGame mutation to accept recap fields. - Enhance game_details_scrape to fetch recap article metadata. - Modify games_scraper to parse and store recap information. - Update GameService to retrieve games using scraper match levels to update fields like location, time, state, city etc if changes occur --- src/models/game.py | 20 +++ src/mutations/create_game.py | 20 ++- src/repositories/game_repository.py | 37 +++- src/scrapers/game_details_scrape.py | 110 ++++++++++-- src/scrapers/games_scraper.py | 270 +++++++++++++++++----------- src/services/game_service.py | 8 + src/types.py | 16 +- src/utils/constants.py | 29 ++- src/utils/convert_to_utc.py | 16 +- src/utils/helpers.py | 43 +++-- 10 files changed, 428 insertions(+), 141 deletions(-) diff --git a/src/models/game.py b/src/models/game.py index 73a7968..65042b6 100644 --- a/src/models/game.py +++ b/src/models/game.py @@ -18,6 +18,10 @@ class Game: - `box_score` The scoring summary of the game (optional) - `score_breakdown` The scoring breakdown of the game (optional) - 'ticket_link' The ticket link for the game (optional) + - `recap_link` The recap article link (optional) + - `recap_article_title` The recap article title (optional) + - `recap_article_image` The recap article image (optional) + - `recap_published_at` The recap article publication date (optional) """ def __init__( @@ -37,6 +41,10 @@ def __init__( team=None, utc_date=None, ticket_link=None, + recap_link=None, + recap_article_title=None, + recap_article_image=None, + recap_published_at=None, ): self.id = id if id else str(ObjectId()) self.city = city @@ -53,6 +61,10 @@ def __init__( self.team = team self.utc_date = utc_date self.ticket_link = ticket_link + self.recap_link = recap_link + self.recap_article_title = recap_article_title + self.recap_article_image = recap_article_image + self.recap_published_at = recap_published_at def to_dict(self): """ @@ -74,6 +86,10 @@ def to_dict(self): "team": self.team, "utc_date": self.utc_date, "ticket_link": self.ticket_link, + "recap_link": self.recap_link, + "recap_article_title": self.recap_article_title, + "recap_article_image": self.recap_article_image, + "recap_published_at": self.recap_published_at, } @staticmethod @@ -97,4 +113,8 @@ def from_dict(data) -> None: team=data.get("team"), utc_date=data.get("utc_date"), ticket_link=data.get("ticket_link"), + recap_link=data.get("recap_link"), + recap_article_title=data.get("recap_article_title"), + recap_article_image=data.get("recap_article_image"), + recap_published_at=data.get("recap_published_at"), ) diff --git a/src/mutations/create_game.py b/src/mutations/create_game.py index 3a52345..0023727 100644 --- a/src/mutations/create_game.py +++ b/src/mutations/create_game.py @@ -13,11 +13,15 @@ class Arguments: result = String(required=False) sport = String(required=True) state = String(required=True) - time = String(required=True) + time = String(required=False) box_score = String(required=False) score_breakdown = String(required=False) utc_date = String(required=False) ticket_link = String(required=False) + recap_link = String(required=False) + recap_article_title = String(required=False) + recap_article_image = String(required=False) + recap_published_at = String(required=False) game = Field(lambda: GameType) @@ -36,7 +40,11 @@ def mutate( box_score=None, score_breakdown=None, utc_date=None, - ticket_link=None + ticket_link=None, + recap_link=None, + recap_article_title=None, + recap_article_image=None, + recap_published_at=None, ): game_data = { "city": city, @@ -51,7 +59,11 @@ def mutate( "box_score": box_score, "score_breakdown": score_breakdown, "utc_date": utc_date, - "ticket_link": ticket_link + "ticket_link": ticket_link, + "recap_link": recap_link, + "recap_article_title": recap_article_title, + "recap_article_image": recap_article_image, + "recap_published_at": recap_published_at, } new_game = GameService.create_game(game_data) - return CreateGame(game=new_game) \ No newline at end of file + return CreateGame(game=new_game) diff --git a/src/repositories/game_repository.py b/src/repositories/game_repository.py index 62a19d2..cc94669 100644 --- a/src/repositories/game_repository.py +++ b/src/repositories/game_repository.py @@ -180,6 +180,39 @@ def find_by_tournament_key_fields(city, date, gender, location, sport, state): return [Game.from_dict(game) for game in games] + @staticmethod + def find_by_scraper_match_levels(date, sport, gender, opponent_id, city, state, location): + """Find a game using the scraper's progressively weaker identity keys.""" + game_collection = db["game"] + base_query = {"date": date, "sport": sport, "gender": gender} + queries = [ + { + **base_query, + "opponent_id": opponent_id, + "city": city, + "state": state, + "location": location, + }, + {**base_query, "opponent_id": opponent_id}, + base_query, + ] + + for level, query in enumerate(queries, start=1): + candidates = list(game_collection.find(query)) + if len(candidates) == 1: + return Game.from_dict(candidates[0]), level + if len(candidates) > 1: + logger.warning( + "Multiple games matched %s %s on %s at match level %s; skipping", + sport, + gender, + date, + level, + ) + return None, level + + return None, None + @staticmethod def find_by_sport(sport): """ @@ -221,7 +254,9 @@ def find_games_by_sport_gender_after_date(sport, gender, after_date=None): } if after_date: - query["utc_date"] = {"$gt": after_date} + query["utc_date"] = { + "$gt": after_date.isoformat() if hasattr(after_date, "isoformat") else after_date + } games = game_collection.find(query) return [Game.from_dict(game) for game in games] diff --git a/src/scrapers/game_details_scrape.py b/src/scrapers/game_details_scrape.py index 5f2f3b1..5c10808 100644 --- a/src/scrapers/game_details_scrape.py +++ b/src/scrapers/game_details_scrape.py @@ -1,8 +1,13 @@ import re +import logging import requests from bs4 import BeautifulSoup +from urllib.parse import urljoin from src.utils.constants import * + +logger = logging.getLogger(__name__) + def clean_name(name): """Strip extra information from player names, keeping only first and last name.""" # try to match firstname, lastname format @@ -25,6 +30,95 @@ def fetch_page(url): response = requests.get(url) return BeautifulSoup(response.text, 'html.parser') + +def fetch_recap_page(url): + response = requests.get(url, headers=HTTP_REQUEST_HEADERS, timeout=30) + response.raise_for_status() + return BeautifulSoup(response.text, "html.parser") + + +def _metadata_content(soup, selector): + tag = soup.select_one(selector) + return tag.get("content") if tag else None + + +def _tag_text_or_datetime(tag): + if not tag: + return None + return tag.get("datetime") or tag.get_text(" ", strip=True) or None + + +def _published_at(soup): + """Extract the article's publication timestamp without empty/related dates.""" + # Older Sidearm stories put the publication timestamp in this story-date + # block without a pubdate attribute. + for tag in soup.select(SIDEARM_STORY_PUBLISHED_TIME_FALLBACK): + value = _tag_text_or_datetime(tag) + if value: + return value + + metadata_value = _metadata_content(soup, 'meta[property="article:published_time"]') + if metadata_value: + return metadata_value + + # Newer stories use pubdate. Ignore empty placeholders and compact dates + # from related-story cards (for example, "09.11.26"). + for tag in soup.select(SIDEARM_STORY_PUBLISHED_TIME): + value = _tag_text_or_datetime(tag) + if value and re.search(r"\b20\d{2}\b", value): + return value + + return None + + +def _first_image_url(soup, base_url): + image = soup.select_one(SIDEARM_STORY_IMAGE) + if image: + image_url = image.get("data-src") or image.get("src") + if image_url: + return urljoin(base_url, image_url) + + # Some Sidearm stories put the responsive image only in a tag. + source = soup.select_one(".sidearm-story-template-media source") + if source: + srcset = source.get("srcset", "").split(",")[0].strip().split(" ")[0] + if srcset: + return urljoin(base_url, srcset) + + metadata_image = _metadata_content(soup, 'meta[property="og:image"]') + return urljoin(base_url, metadata_image) if metadata_image else None + + +def scrape_sidearm_story_recap(url): + """Scrape the article metadata from a Sidearm recap page. + + ``None`` means the page could not be fetched or parsed. A dictionary with + nullable fields means the page was fetched successfully, which lets the + schedule scraper preserve existing article data on transient failures. + """ + if not url: + return None + + try: + soup = fetch_recap_page(url) + except (requests.RequestException, ValueError) as exc: + logger.warning("Unable to fetch recap page %s: %s", url, exc) + return None + except Exception as exc: + logger.exception("Unexpected error fetching recap page %s: %s", url, exc) + return None + + headline = soup.select_one(SIDEARM_STORY_HEADLINE) + return { + "recap_article_title": ( + headline.get_text(" ", strip=True) + if headline + else _metadata_content(soup, 'meta[property="og:title"]') + ), + "recap_article_image": _first_image_url(soup, url), + "recap_published_at": _published_at(soup), + } + def extract_teams_and_scores(box_score_section, sport): score_table = box_score_section.find(TAG_TABLE, class_=CLASS_SIDEARM_TABLE) team_names = [] @@ -79,8 +173,6 @@ def soccer_summary(box_score_section): 'cor_score': cornell_score, 'opp_score': opp_score }) - if not summary: - summary = [{"message": "No scoring events in this game."}] return summary def football_summary(box_score_section): @@ -105,8 +197,6 @@ def football_summary(box_score_section): 'cor_score': cornell_score, 'opp_score': opp_score }) - if not summary: - summary = [{"message": "No scoring events in this game."}] return summary def hockey_summary(box_score_section): @@ -139,8 +229,6 @@ def hockey_summary(box_score_section): 'opp_score': opp_score, 'description': f"Scored by {scorer}. Assisted by {assist}." }) - if not summary: - summary = [{"message": "No scoring events in this game."}] return summary def field_hockey_summary(box_score_section): @@ -167,8 +255,6 @@ def field_hockey_summary(box_score_section): 'cor_score': cornell_score, 'opp_score': opp_score }) - if not summary: - summary = [{"message": "No scoring events in this game."}] return summary def lacrosse_summary(box_score_section): @@ -201,8 +287,6 @@ def lacrosse_summary(box_score_section): 'cor_score': cor_score, 'opp_score': opp_score, }) - if not summary: - summary = [{"message": "No scoring events in this game."}] return summary def baseball_summary(box_score_section): @@ -225,8 +309,6 @@ def baseball_summary(box_score_section): 'cor_score': cor_score, 'opp_score': opp_score }) - if not summary: - summary = [{"message": "No scoring events in this game."}] return summary # def basketball_summary(box_score_section): @@ -288,7 +370,7 @@ def scrape_game(url, sport): return { 'teams': team_names, 'scores': scores, - 'scoring_summary': scoring_summary or [{"message": "No scoring events in this game."}] + 'scoring_summary': scoring_summary } - return {"error": "Sport parser not found"} \ No newline at end of file + return {"error": "Sport parser not found"} diff --git a/src/scrapers/games_scraper.py b/src/scrapers/games_scraper.py index 818760c..6b5cecb 100644 --- a/src/scrapers/games_scraper.py +++ b/src/scrapers/games_scraper.py @@ -1,14 +1,18 @@ import requests from bs4 import BeautifulSoup -from src.services import GameService, TeamService from src.utils.convert_to_utc import convert_to_utc from src.utils.constants import * -from src.scrapers.game_details_scrape import scrape_game -from src.utils.helpers import get_dominant_color, normalize_game_data, is_tournament_placeholder_team, is_cornell_loss +from src.scrapers.game_details_scrape import scrape_game, scrape_sidearm_story_recap +from src.utils.helpers import get_dominant_color, normalize_game_data, is_tournament_placeholder_team, is_cornell_loss, normalize_placeholder import base64 +import logging import re -from src.database import db import threading +from urllib.parse import urljoin + + +logger = logging.getLogger(__name__) +RECAP_FIELDS = ["recap_article_title", "recap_article_image", "recap_published_at"] def extract_season_years(page_title): @@ -40,6 +44,74 @@ def infer_game_year(date_text, season_years): return second_year return first_year + +def absolute_url(link): + return urljoin(BASE_URL.rstrip("/") + "/", link) if link else None + + +def parse_game_links(game_item): + links = {} + for name, selector in { + "box_score_link": BOX_SCORE_TAG, + "recap_link": RECAP_TAG, + "ticket_link": GAME_TICKET_LINK, + }.items(): + tag = game_item.select_one(selector) + links[name] = absolute_url(tag.get("href")) if tag and tag.get("href") else None + return links + + +def parse_schedule_location(location_text): + if not location_text or str(location_text).strip().casefold() in {"tba", "tbd"}: + return "TBA", "TBA", "TBA" + + parts = re.split(r"\s*/\s*|\s*\n\s*", str(location_text).strip(), maxsplit=1) + geo_location = parts[0].strip() + location = parts[1].strip() if len(parts) > 1 else "TBA" + if "," in geo_location: + city, state = [part.strip() for part in geo_location.split(",", 1)] + else: + city = state = geo_location + return tuple(normalize_placeholder(value) for value in (city, state, location)) + + +def parse_schedule_date_and_time(game_item): + """Read the start date and time without treating an end date as a time. + + Sidearm uses the same date block for single-day and multi-day events. For + example, a tournament row can contain ``Nov 12``, ``Nov 15`` and ``TBA`` + as separate spans. The old adjacent-sibling selector returned ``Nov 15`` + as the time in that case. + """ + date_block = game_item.select_one(SCHEDULE_DATE_BLOCK_TAG) + if not date_block: + return "", "TBA" + + spans = date_block.find_all("span") + date_text = spans[0].get_text(" ", strip=True) if spans else "" + time_tag = next( + ( + span + for span in spans[1:] + if "enddate" not in {name.casefold() for name in span.get("class", [])} + ), + None, + ) + time_text = normalize_placeholder( + time_tag.get_text(" ", strip=True) if time_tag else None + ) + return date_text, time_text + + +def _recap(recap_link): + if not recap_link: + return {field: None for field in RECAP_FIELDS} | {"success": True} + recap = scrape_sidearm_story_recap(recap_link) + if not recap or not any(recap.get(field) for field in RECAP_FIELDS): + return {field: None for field in RECAP_FIELDS} | {"success": False} + return recap | {"success": True} + + def fetch_game_schedule(): """ Scrape the game schedule from the given URLs in parallel using threads. @@ -71,7 +143,13 @@ def parse_schedule_page(url, sport, gender): sport (str): The sport of the games. gender (str): The gender of the games. """ - response = requests.get(url) + try: + response = requests.get(url, headers=HTTP_REQUEST_HEADERS, timeout=30) + response.raise_for_status() + except Exception as exc: + logger.warning("Unable to fetch schedule %s: %s", url, exc) + return + soup = BeautifulSoup(response.content, "html.parser") page_title = soup.title.text.strip() if soup.title else "" @@ -82,31 +160,25 @@ def parse_schedule_page(url, sport, gender): game_data["gender"] = gender game_data["sport"] = sport - opponent_name_tag = game_item.select_one(OPPONENT_NAME_TAG_A) - opponent_name = ( - opponent_name_tag.text.strip() - if opponent_name_tag - else game_item.select_one(OPPONENT_NAME_TAG).text.strip() + opponent_name_tag = game_item.select_one(OPPONENT_NAME_TAG_A) or game_item.select_one(OPPONENT_NAME_TAG) + game_data["opponent_name"] = normalize_placeholder( + opponent_name_tag.text.strip() if opponent_name_tag else None ) - game_data["opponent_name"] = opponent_name opponent_logo_tag = game_item.select_one(OPPONENT_LOGO_TAG) opponent_logo = ( - opponent_logo_tag[OPPONENT_LOGO_URL_ATTR] if opponent_logo_tag else None - ) - game_data["opponent_logo"] = ( - BASE_URL + opponent_logo if opponent_logo else None + opponent_logo_tag.get(OPPONENT_LOGO_URL_ATTR) + or opponent_logo_tag.get("src") + if opponent_logo_tag else None ) + game_data["opponent_logo"] = absolute_url(opponent_logo) - date_tag = game_item.select_one(DATE_TAG) - if date_tag: - date_text = date_tag.get_text(strip=True) - else: - date_text = "" + date_text, time_text = parse_schedule_date_and_time(game_item) + + if not date_text: + logger.warning("Skipping %s row without a date", sport) + continue - time_tag = game_item.select_one(TIME_TAG) - time_text = time_tag.text.strip() if time_tag else None - game_year = infer_game_year(date_text, season_years) # keep old date field for now @@ -121,21 +193,24 @@ def parse_schedule_page(url, sport, gender): game_data["time"] = time_text location_tag = game_item.select_one(LOCATION_TAG) - game_data["location"] = location_tag.text.strip() if location_tag else None + game_data["location"] = location_tag.get_text("\n", strip=True) if location_tag else None result_tag = game_item.select_one(RESULT_TAG) if result_tag: game_data["result"] = result_tag.text.strip().replace("\n", "") else: game_data["result"] = None - + + links = parse_game_links(game_item) box_score_tag = game_item.select_one(BOX_SCORE_TAG) + game_data["_box_score_scrape_succeeded"] = True if box_score_tag: box_score_link = box_score_tag["href"] game_details = scrape_game(f"{BASE_URL}{box_score_link}", sport.lower()) if game_details.get('error') == 'Sport parser not found': game_data["box_score"] = None game_data["score_breakdown"] = None + game_data["_box_score_scrape_succeeded"] = False else: game_data["box_score"] = game_details.get("scoring_summary") game_data["score_breakdown"] = game_details.get("scores") @@ -144,45 +219,60 @@ def parse_schedule_page(url, sport, gender): location_data = game_data["location"].split("\n") if game_data["location"] else [""] geo_location = location_data[0] is_home_game = "Ithaca" in geo_location - + if is_home_game and game_data["box_score"]: for event in game_data["box_score"]: if "cor_score" in event and "opp_score" in event: event["cor_score"], event["opp_score"] = event["opp_score"], event["cor_score"] - else: game_data["box_score"] = None game_data["score_breakdown"] = None - + + recap = _recap(links["recap_link"]) + game_data["recap_link"] = links["recap_link"] + for field in RECAP_FIELDS: + game_data[field] = recap[field] + game_data["_recap_scrape_succeeded"] = recap["success"] + ticket_link_tag = game_item.select_one(GAME_TICKET_LINK) ticket_link = ( ticket_link_tag["href"] if ticket_link_tag else None ) game_data["ticket_link"] = ( - ticket_link if ticket_link else None + absolute_url(ticket_link) if ticket_link else None ) process_game_data(game_data) +def _detail_updates(game_data): + updates = {} + if game_data.get("_box_score_scrape_succeeded", "box_score" in game_data): + updates["box_score"] = game_data.get("box_score") + updates["score_breakdown"] = game_data.get("score_breakdown") + if game_data.get("_recap_scrape_succeeded", "recap_link" in game_data): + updates["recap_link"] = game_data.get("recap_link") + if game_data.get("recap_link"): + updates.update({field: game_data.get(field) for field in RECAP_FIELDS if game_data.get(field) is not None}) + else: + updates.update({field: None for field in RECAP_FIELDS}) + return updates + + def process_game_data(game_data): """ Process the game data and store it in the database. Args: - game_data (dict): A dictionary containing the game data. + game_data (dict): A dictionary containing the data for a game. """ - - game_data = normalize_game_data(game_data) - location_data = game_data["location"].split("\n") - geo_location = location_data[0] - if (",") not in geo_location: - city = geo_location - state = geo_location + from src.services import GameService, TeamService + + if "city" in game_data or "state" in game_data: + city, state, location = game_data.get("city"), game_data.get("state"), game_data.get("location") else: - parts = [part.strip() for part in geo_location.split(",")] - city = parts[0] - state = parts[-1] - location = location_data[1] if len(location_data) > 1 else None + city, state, location = parse_schedule_location(game_data.get("location")) + game_data.update(city=city, state=state, location=location) + game_data = normalize_game_data(game_data) team = TeamService.get_team_by_name(game_data["opponent_name"]) if not team: @@ -194,7 +284,7 @@ def process_game_data(game_data): encoded_opponent_logo = "" if game_data["opponent_logo"]: try: - response = requests.get(game_data["opponent_logo"]) + response = requests.get(game_data["opponent_logo"], headers=HTTP_REQUEST_HEADERS, timeout=30) response.raise_for_status() encoded_opponent_logo = base64.b64encode(response.content).decode('utf-8') except Exception as e: @@ -208,12 +298,10 @@ def process_game_data(game_data): team = TeamService.create_team(team_data) # ISO format - utc_date_str = game_data["utc_date"].isoformat() if game_data["utc_date"] else None - - game_time = game_data["time"] - if game_time is None: - game_time = "TBD" + utc_date_obj = game_data["utc_date"] + utc_date_str = utc_date_obj.isoformat() if hasattr(utc_date_obj, "isoformat") else utc_date_obj + game_time = normalize_placeholder(game_data.get("time")) is_home_game = "Ithaca" in city # make sure cornell is first in score breakdown - switch order on home games @@ -233,7 +321,7 @@ def process_game_data(game_data): break # Compare with score breakdown - if final_box_cor_score and len(game_data["score_breakdown"]) >= 2: + if final_box_cor_score is not None and len(game_data["score_breakdown"]) >= 2: cor_final = game_data["score_breakdown"][0][-1] opp_final = game_data["score_breakdown"][1][-1] @@ -241,70 +329,48 @@ def process_game_data(game_data): if str(final_box_cor_score) != str(cor_final) or str(final_box_opp_score) != str(opp_final): game_data["score_breakdown"] = game_data["score_breakdown"][::-1] - # Try to find by tournament key fields to handle placeholder teams - curr_game = GameService.get_game_by_tournament_key_fields( - city, + curr_game, match_level = GameService.get_game_by_scraper_match_levels( game_data["date"], + game_data["sport"], game_data["gender"], + team.id, + city, + state, location, - game_data["sport"], - state ) - - # If no tournament game found, try the regular lookup with opponent_id - if not curr_game: - curr_game = GameService.get_game_by_key_fields( - city, - game_data["date"], - game_data["gender"], - location, - team.id, - game_data["sport"], - state - ) + if curr_game is None and match_level is not None: + return None - if isinstance(curr_game, list): - if curr_game: - curr_game = curr_game[0] - else: - curr_game = None + updates = { + "time": game_time, + "result": game_data["result"], + "utc_date": utc_date_str, + "city": city, + "location": location, + "state": state, + "opponent_id": team.id, + "ticket_link": game_data["ticket_link"], + **_detail_updates(game_data), + } if curr_game: - updates = { - "time": game_time, - "result": game_data["result"], - "box_score": game_data["box_score"], - "score_breakdown": game_data["score_breakdown"], - "utc_date": utc_date_str, - "city": city, - "location": location, - "state": state, - "ticket_link": game_data["ticket_link"] - } - current_team = TeamService.get_team_by_id(curr_game.opponent_id) if current_team and is_tournament_placeholder_team(current_team.name): - updates["opponent_id"] = team.id - - if is_cornell_loss(game_data["result"]) and game_data["utc_date"]: - GameService.handle_tournament_loss(game_data["sport"], game_data["gender"], game_data["utc_date"]) - + if is_cornell_loss(game_data["result"]) and utc_date_obj: + GameService.handle_tournament_loss(game_data["sport"], game_data["gender"], utc_date_obj) GameService.update_game(curr_game.id, updates) - return - - game_data = { - "city": city, + return curr_game.id + + create_data = { + **updates, "date": game_data["date"], "gender": game_data["gender"], - "location": location, - "opponent_id": team.id, - "result": game_data["result"], "sport": game_data["sport"], - "state": state, - "time": game_time, - "box_score": game_data["box_score"], - "score_breakdown": game_data["score_breakdown"], - "utc_date": utc_date_str, - "ticket_link": game_data["ticket_link"] + "box_score": updates.get("box_score"), + "score_breakdown": updates.get("score_breakdown"), + "recap_link": game_data.get("recap_link"), + "recap_article_title": updates.get("recap_article_title"), + "recap_article_image": updates.get("recap_article_image"), + "recap_published_at": updates.get("recap_published_at"), } - - GameService.create_game(game_data) \ No newline at end of file + created = GameService.create_game(create_data) + return created.id if created else None diff --git a/src/services/game_service.py b/src/services/game_service.py index c7c4721..3f830aa 100644 --- a/src/services/game_service.py +++ b/src/services/game_service.py @@ -92,6 +92,14 @@ def get_game_by_tournament_key_fields(city, date, gender, location, sport, state city, date, gender, location, sport, state ) + @staticmethod + def get_game_by_scraper_match_levels( + date, sport, gender, opponent_id, city, state, location + ): + return GameRepository.find_by_scraper_match_levels( + date, sport, gender, opponent_id, city, state, location + ) + @staticmethod def get_games_by_sport(sport): """ diff --git a/src/types.py b/src/types.py index 7eb8fbe..dfbd069 100644 --- a/src/types.py +++ b/src/types.py @@ -89,6 +89,10 @@ class GameType(ObjectType): - `box_score`: The box score of the game. - `score_breakdown`: The score breakdown of the game. - `ticket_link`: The ticket link of the game. (optional) + - `recap_link`: The recap article link. (optional) + - `recap_article_title`: The recap article title. (optional) + - `recap_article_image`: The recap article image. (optional) + - `recap_published_at`: The recap article publication date. (optional) """ id = String(required=False) @@ -106,8 +110,12 @@ class GameType(ObjectType): team = Field(TeamType, required=False) utc_date = String(required=False) ticket_link = String(required=False) + recap_link = String(required=False) + recap_article_title = String(required=False) + recap_article_image = String(required=False) + recap_published_at = String(required=False) def __init__( - self, id, city, date, gender, location, opponent_id, result, sport, state, time, box_score=None, score_breakdown=None, utc_date=None, ticket_link=None + self, id, city, date, gender, location, opponent_id, result, sport, state, time, box_score=None, score_breakdown=None, utc_date=None, ticket_link=None, recap_link=None, recap_article_title=None, recap_article_image=None, recap_published_at=None ): self.id = id self.city = city @@ -123,6 +131,10 @@ def __init__( self.score_breakdown = score_breakdown self.utc_date = utc_date self.ticket_link = ticket_link + self.recap_link = recap_link + self.recap_article_title = recap_article_title + self.recap_article_image = recap_article_image + self.recap_published_at = recap_published_at @staticmethod def team_to_team_type(team_obj): if team_obj is None: @@ -199,4 +211,4 @@ class ArticleType(ObjectType): def __init__(self, **kwargs): for key, value in kwargs.items(): - setattr(self, key, value) \ No newline at end of file + setattr(self, key, value) diff --git a/src/utils/constants.py b/src/utils/constants.py index 38c2ae7..639f349 100644 --- a/src/utils/constants.py +++ b/src/utils/constants.py @@ -9,6 +9,13 @@ # Base URL BASE_URL = "https://cornellbigred.com" +# Use the same headers for schedule and detail requests. Sidearm pages can +# return an incomplete response to requests that do not look like a browser. +HTTP_REQUEST_HEADERS = { + "User-Agent": "Mozilla/5.0 (compatible; CornellSportsScraper/1.0)", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", +} + # The tag for each game GAME_TAG = ".sidearm-schedule-game" @@ -24,11 +31,12 @@ # The attribute for the opponent image URL OPPONENT_LOGO_URL_ATTR = "data-src" -# The tag for each date -DATE_TAG = ".sidearm-schedule-game-opponent-date span" +# The date/time block for each schedule row +SCHEDULE_DATE_BLOCK_TAG = ".sidearm-schedule-game-opponent-date" -# The tag for each time -TIME_TAG = ".sidearm-schedule-game-opponent-date span + span" +# The first span in the date/time block is the start date. The remaining spans +# can contain a time, an end date for a multi-day event, or a display-only span. +DATE_TAG = f"{SCHEDULE_DATE_BLOCK_TAG} span" # The tag for each location LOCATION_TAG = ".sidearm-schedule-game-location" @@ -45,9 +53,18 @@ # The tag for the box score BOX_SCORE_TAG = ".sidearm-schedule-game-links-boxscore a" +# The tag for the recap article link +RECAP_TAG = ".sidearm-schedule-game-links-recap a" + # The tag for the game ticket link GAME_TICKET_LINK = ".sidearm-schedule-game-links-tickets a" +# Sidearm story article selectors +SIDEARM_STORY_HEADLINE = "h1.sidearm-story-template-headline" +SIDEARM_STORY_PUBLISHED_TIME = "time[pubdate]" +SIDEARM_STORY_PUBLISHED_TIME_FALLBACK = ".sidearm-story-template-date time" +SIDEARM_STORY_IMAGE = ".sidearm-story-template-media img" + # HTML Tags TAG_TABLE = 'table' TAG_SECTION = 'section' @@ -86,7 +103,9 @@ LABEL_SCORING_SUMMARY = 'Scoring Summary' LABEL_CU = 'CU' -# The dictionary mapping sports urls to gender +# The dictionary mapping every Cornell sports schedule URL to sport and gender. +# A sport can still be scheduled here when its box-score parser is unavailable; +# the schedule scraper will retain the game without detail fields. SPORT_URLS = { "baseball": {"sport": "Baseball", "gender": "Mens"}, "mens-basketball": {"sport": "Basketball", "gender": "Mens"}, diff --git a/src/utils/convert_to_utc.py b/src/utils/convert_to_utc.py index 8a7fc2c..9674d33 100644 --- a/src/utils/convert_to_utc.py +++ b/src/utils/convert_to_utc.py @@ -15,8 +15,18 @@ def parse_time_string(time_str): # sometimes written as p.m. time_str = time_str.replace("p.m.", "pm").replace("a.m.", "am") + # A date such as "Nov 15 (Sun)" is not a time. Require either a + # meridiem or a colon-formatted clock value before parsing. + has_meridiem = re.search(r'\b(?:am|pm)\b', time_str) + has_colon_time = re.search(r'(? Date: Sat, 12 Sep 2026 17:38:51 -0400 Subject: [PATCH 2/4] Potential fix for pull request finding 'CodeQL / Clear-text logging of sensitive information' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- src/repositories/game_repository.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/repositories/game_repository.py b/src/repositories/game_repository.py index cc94669..b24043f 100644 --- a/src/repositories/game_repository.py +++ b/src/repositories/game_repository.py @@ -203,10 +203,7 @@ def find_by_scraper_match_levels(date, sport, gender, opponent_id, city, state, return Game.from_dict(candidates[0]), level if len(candidates) > 1: logger.warning( - "Multiple games matched %s %s on %s at match level %s; skipping", - sport, - gender, - date, + "Multiple games matched at match level %s; skipping", level, ) return None, level From 51d85718601652be096da51ca7b08b3c1b85c272 Mon Sep 17 00:00:00 2001 From: claiireyu Date: Sat, 12 Sep 2026 21:29:23 -0400 Subject: [PATCH 3/4] Enhance URL validation and safety checks in scrapers and repositories --- src/repositories/game_repository.py | 3 +- src/scrapers/game_details_scrape.py | 35 ++++++++++++++++++- src/scrapers/games_scraper.py | 54 ++++++++++++++++++++--------- src/utils/constants.py | 7 ++++ src/utils/helpers.py | 30 ++++++++++++++++ 5 files changed, 109 insertions(+), 20 deletions(-) diff --git a/src/repositories/game_repository.py b/src/repositories/game_repository.py index b24043f..bc8076e 100644 --- a/src/repositories/game_repository.py +++ b/src/repositories/game_repository.py @@ -182,7 +182,7 @@ def find_by_tournament_key_fields(city, date, gender, location, sport, state): @staticmethod def find_by_scraper_match_levels(date, sport, gender, opponent_id, city, state, location): - """Find a game using the scraper's progressively weaker identity keys.""" + """Find a game without matching unrelated opponents by date alone.""" game_collection = db["game"] base_query = {"date": date, "sport": sport, "gender": gender} queries = [ @@ -194,7 +194,6 @@ def find_by_scraper_match_levels(date, sport, gender, opponent_id, city, state, "location": location, }, {**base_query, "opponent_id": opponent_id}, - base_query, ] for level, query in enumerate(queries, start=1): diff --git a/src/scrapers/game_details_scrape.py b/src/scrapers/game_details_scrape.py index 5c10808..56b87ae 100644 --- a/src/scrapers/game_details_scrape.py +++ b/src/scrapers/game_details_scrape.py @@ -4,6 +4,7 @@ from bs4 import BeautifulSoup from urllib.parse import urljoin from src.utils.constants import * +from src.utils.helpers import is_allowed_url logger = logging.getLogger(__name__) @@ -27,11 +28,16 @@ def clean_name(name): return cleaned def fetch_page(url): - response = requests.get(url) + if not is_allowed_url(url): + raise ValueError(f"Unapproved box score URL: {url}") + response = requests.get(url, headers=HTTP_REQUEST_HEADERS, timeout=30) + response.raise_for_status() return BeautifulSoup(response.text, 'html.parser') def fetch_recap_page(url): + if not is_allowed_url(url): + raise ValueError(f"Unapproved recap URL: {url}") response = requests.get(url, headers=HTTP_REQUEST_HEADERS, timeout=30) response.raise_for_status() return BeautifulSoup(response.text, "html.parser") @@ -311,6 +317,32 @@ def baseball_summary(box_score_section): }) return summary +def softball_summary(box_score_section): + summary = [] + scoring_section = box_score_section.find(TAG_SECTION, {ATTR_ARIA_LABEL: LABEL_SCORING_SUMMARY}) + if scoring_section: + scoring_rows = scoring_section.find(TAG_TBODY) + if scoring_rows: + for row in scoring_rows.find_all(TAG_TR): + cells = row.find_all(TAG_TD) + team = cells[0].find(TAG_IMG)[ATTR_ALT] + inning = cells[3].get_text(strip=True) + description = cells[4] + span = description.find(TAG_SPAN) + if span: + span.extract() + summary.append({ + 'team': team, + 'period': inning, + 'inning': inning, + 'description': description.get_text(strip=True), + 'cor_score': int(cells[5].get_text(strip=True) or 0), + 'opp_score': int(cells[6].get_text(strip=True) or 0), + }) + if not summary: + summary = [{"message": "No scoring events in this game."}] + return summary + # def basketball_summary(box_score_section): # summary = [] # scoring_section = box_score_section.find(TAG_SECTION, {ATTR_ARIA_LABEL: LABEL_SCORING_SUMMARY}) @@ -354,6 +386,7 @@ def scrape_game(url, sport): 'field hockey': (lambda: extract_teams_and_scores(box_score_section, 'field hockey'), field_hockey_summary), 'lacrosse': (lambda: extract_teams_and_scores(box_score_section, 'lacrosse'), lacrosse_summary), 'baseball': (lambda: extract_teams_and_scores(box_score_section, 'baseball'), baseball_summary), + 'softball': (lambda: extract_teams_and_scores(box_score_section, 'softball'), softball_summary), 'basketball': (lambda: extract_teams_and_scores(box_score_section, 'basketball'), lambda _: []), } diff --git a/src/scrapers/games_scraper.py b/src/scrapers/games_scraper.py index 6b5cecb..267a051 100644 --- a/src/scrapers/games_scraper.py +++ b/src/scrapers/games_scraper.py @@ -3,7 +3,15 @@ from src.utils.convert_to_utc import convert_to_utc from src.utils.constants import * from src.scrapers.game_details_scrape import scrape_game, scrape_sidearm_story_recap -from src.utils.helpers import get_dominant_color, normalize_game_data, is_tournament_placeholder_team, is_cornell_loss, normalize_placeholder +from src.utils.helpers import ( + get_dominant_color, + is_cornell_loss, + is_allowed_url, + is_tournament_placeholder_team, + normalize_game_data, + normalize_placeholder, + safe_absolute_url, +) import base64 import logging import re @@ -46,7 +54,7 @@ def infer_game_year(date_text, season_years): def absolute_url(link): - return urljoin(BASE_URL.rstrip("/") + "/", link) if link else None + return safe_absolute_url(link) def parse_game_links(game_item): @@ -57,7 +65,11 @@ def parse_game_links(game_item): "ticket_link": GAME_TICKET_LINK, }.items(): tag = game_item.select_one(selector) - links[name] = absolute_url(tag.get("href")) if tag and tag.get("href") else None + href = tag.get("href") if tag else None + if name == "ticket_link": + links[name] = urljoin(BASE_URL, href) if href else None + else: + links[name] = absolute_url(href) return links @@ -202,16 +214,26 @@ def parse_schedule_page(url, sport, gender): game_data["result"] = None links = parse_game_links(game_item) - box_score_tag = game_item.select_one(BOX_SCORE_TAG) - game_data["_box_score_scrape_succeeded"] = True - if box_score_tag: - box_score_link = box_score_tag["href"] - game_details = scrape_game(f"{BASE_URL}{box_score_link}", sport.lower()) - if game_details.get('error') == 'Sport parser not found': + box_score_link = links["box_score_link"] + game_data["_box_score_scrape_succeeded"] = False + if box_score_link: + try: + game_details = scrape_game(box_score_link, sport.lower()) + except Exception as exc: + logger.warning("Unable to scrape box score %s: %s", box_score_link, exc) + game_details = None + + if not isinstance(game_details, dict) or game_details.get("error"): + if isinstance(game_details, dict) and game_details.get("error"): + logger.warning( + "Box score scrape failed for %s: %s", + box_score_link, + game_details["error"], + ) game_data["box_score"] = None game_data["score_breakdown"] = None - game_data["_box_score_scrape_succeeded"] = False else: + game_data["_box_score_scrape_succeeded"] = True game_data["box_score"] = game_details.get("scoring_summary") game_data["score_breakdown"] = game_details.get("scores") @@ -234,13 +256,7 @@ def parse_schedule_page(url, sport, gender): game_data[field] = recap[field] game_data["_recap_scrape_succeeded"] = recap["success"] - ticket_link_tag = game_item.select_one(GAME_TICKET_LINK) - ticket_link = ( - ticket_link_tag["href"] if ticket_link_tag else None - ) - game_data["ticket_link"] = ( - absolute_url(ticket_link) if ticket_link else None - ) + game_data["ticket_link"] = links["ticket_link"] process_game_data(game_data) @@ -274,6 +290,10 @@ def process_game_data(game_data): game_data.update(city=city, state=state, location=location) game_data = normalize_game_data(game_data) + if game_data.get("opponent_logo") and not is_allowed_url(game_data["opponent_logo"]): + logger.warning("Skipping unapproved opponent logo URL: %s", game_data["opponent_logo"]) + game_data["opponent_logo"] = None + team = TeamService.get_team_by_name(game_data["opponent_name"]) if not team: color = ( diff --git a/src/utils/constants.py b/src/utils/constants.py index 639f349..b3843e4 100644 --- a/src/utils/constants.py +++ b/src/utils/constants.py @@ -65,6 +65,13 @@ SIDEARM_STORY_PUBLISHED_TIME_FALLBACK = ".sidearm-story-template-date time" SIDEARM_STORY_IMAGE = ".sidearm-story-template-media img" +# Hosts used by schedule links and Cornell/Sidearm-hosted images. +ALLOWED_URL_SCHEMES = frozenset({"http", "https"}) +ALLOWED_URL_HOSTS = frozenset({ + "cornellbigred.com", + "dxbhsrqyrr690.cloudfront.net", +}) + # HTML Tags TAG_TABLE = 'table' TAG_SECTION = 'section' diff --git a/src/utils/helpers.py b/src/utils/helpers.py index d1a158f..498bed8 100644 --- a/src/utils/helpers.py +++ b/src/utils/helpers.py @@ -4,6 +4,9 @@ from io import BytesIO from collections import Counter import re +from urllib.parse import urljoin, urlparse + +from src.utils.constants import ALLOWED_URL_HOSTS, ALLOWED_URL_SCHEMES, BASE_URL PLACEHOLDER_VALUES = { @@ -13,6 +16,29 @@ } +def is_allowed_url(url): + """Allow only HTTP(S) URLs hosted by approved Cornell/Sidearm domains.""" + if not url: + return False + parsed = urlparse(str(url)) + hostname = (parsed.hostname or "").casefold().rstrip(".") + return ( + parsed.scheme.casefold() in ALLOWED_URL_SCHEMES + and hostname in ALLOWED_URL_HOSTS + ) + + +def safe_absolute_url(link, base_url=BASE_URL): + """Resolve a link and return it only when its destination is approved.""" + if not link: + return None + normalized = urljoin(base_url, str(link)) + if not is_allowed_url(normalized): + logging.warning("Skipping unapproved URL: %s", normalized) + return None + return normalized + + def normalize_placeholder(value, fallback="TBA"): """Return a stable value for blank/unknown source fields.""" if value is None: @@ -38,6 +64,10 @@ def get_dominant_color(image_url, white_threshold=200, black_threshold=50): """ default_color = "#000000" + if not is_allowed_url(image_url): + logging.warning("Skipping unapproved image URL: %s", image_url) + return default_color + try: response = requests.get(image_url, timeout=30) response.raise_for_status() From 0d49b7a2fdc626550a2380252ea9137b786f8d27 Mon Sep 17 00:00:00 2001 From: claiireyu Date: Sat, 12 Sep 2026 21:41:34 -0400 Subject: [PATCH 4/4] Add URL scheme validation for ticket links in game scraper --- src/scrapers/games_scraper.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/scrapers/games_scraper.py b/src/scrapers/games_scraper.py index 267a051..da6563f 100644 --- a/src/scrapers/games_scraper.py +++ b/src/scrapers/games_scraper.py @@ -16,7 +16,7 @@ import logging import re import threading -from urllib.parse import urljoin +from urllib.parse import urljoin, urlparse logger = logging.getLogger(__name__) @@ -67,7 +67,13 @@ def parse_game_links(game_item): tag = game_item.select_one(selector) href = tag.get("href") if tag else None if name == "ticket_link": - links[name] = urljoin(BASE_URL, href) if href else None + ticket_url = urljoin(BASE_URL, href) if href else None + if ticket_url and urlparse(ticket_url).scheme.casefold() in ALLOWED_URL_SCHEMES: + links[name] = ticket_url + else: + if ticket_url: + logger.warning("Skipping ticket URL with unapproved scheme: %s", ticket_url) + links[name] = None else: links[name] = absolute_url(href) return links