diff --git a/.gitignore b/.gitignore index b529cf63..b91940f1 100644 --- a/.gitignore +++ b/.gitignore @@ -341,7 +341,6 @@ spark-2.3.1-bin-hadoop2.7/ !.env.vault NFL-Stats/ -SportsRadar/ nextjs-container/ /backend-container/src/Schema/NFL_v7/*.py /backend-container/src/Schema/NFL_v7/*.json diff --git a/backend-container/src/apimappings/LeagueHierarchy.py b/backend-container/src/apimappings/LeagueHierarchy.py index 2266a3d4..e3cc0ef1 100644 --- a/backend-container/src/apimappings/LeagueHierarchy.py +++ b/backend-container/src/apimappings/LeagueHierarchy.py @@ -1,310 +1,307 @@ -import sys -sys.path.append('..') -import requests, pandas, json -from flask import Blueprint, jsonify, Flask -import os, time -if not hasattr(os, 'add_dll_directory'): - def add_dll_directory(path): - pass -from datetime import datetime -from uuid import UUID -from src.models.franchise_info import(FranchiseInfo) -from src.models.venue_info import(venue1, location, VenueInfo) -from src.models.leaguehierarchy import( teams, division, conference, league,typeleague, LeagueHierarchy) -from src.models.team_info import(coach, rgb_color, team_color, team, TeamInfo) -from src.models.changelog import ChangelogEntry # Import the ChangelogEntry model -from mongoengine import DoesNotExist, DecimalField, EmbeddedDocumentField, Document, StringField, UUIDField, IntField, BooleanField, DateTimeField, EmbeddedDocument, EmbeddedDocumentListField -from bson import ObjectId -bp = Blueprint('LeagueHierarchy', __name__) -@bp.route('/LeagueHierarchy', methods=['GET']) -def fetchandsaveLeagueHierarchy(): - #print("LeagueHierarchy") - url = os.getenv('LEAGUE_HIERARCHY_API_URL') - print(datetime.now(), "Requesting URL:", url) - response = requests.get(url) - print("Response status code:", response.status_code) - if response.status_code != 200: - return f"Call Seasons Successfully{response.status_code}" - data = response.json() - venue_info_dict = extract_venue_info(data) - league_hierarchy_dict = extract_league_hierarchy(data) - franchise_info_dict = extract_franchise_info(data) - team_info_dict = extract_team_info(data) - mapped_venues = map_venue_info(venue_info_dict) - mapped_leaguehierarchy = map_league_hierarchy(league_hierarchy_dict) - mapped_franchises = map_franchise_info(franchise_info_dict) - mapped_teams = map_team_info(team_info_dict) - save_to_database(mapped_venues, mapped_leaguehierarchy, mapped_franchises, mapped_teams) - print("League Hierarchy data fetched and saved successfully.") - return "League Hierarchy data fetched and saved successfully." -def extract_venue_info(data): - venue_info_dict = {} - for conference_data in data['conferences']: - for division_data in conference_data['divisions']: - for team_data in division_data['teams']: - venue_info = { - 'id': team_data['venue']['id'], - 'name': team_data['venue']['name'], - 'city': team_data['venue']['city'], - 'state': team_data['venue'].get('state'), - } - venue_info['country'] = team_data['venue'].get('country') - venue_info['zip'] = team_data['venue'].get('zip') - venue_info['address'] = team_data['venue']['address'] - venue_info['capacity'] = team_data['venue']['capacity'] - venue_info['surface'] = team_data['venue']['surface'] - venue_info['roof_type'] = team_data['venue']['roof_type'] - venue_info['sr_id'] = team_data['venue']['sr_id'] - venue_info['lat'] = team_data.get('venue', {}).get('location', {}).get('lat', None) - venue_info['lng'] = team_data.get('venue', {}).get('location', {}).get('lng', None) - venue_info_dict[team_data['venue']['id']] = venue_info - #print("venue_info_dict:", venue_info_dict) - return venue_info_dict -def map_venue_info(venue_info_dict): - mapped_venues = {} - for venue_id, venue_details in venue_info_dict.items(): - venue_embedded = venue1( - id=venue_details['id'], - address=venue_details['address'], - capacity=venue_details['capacity'], - city=venue_details['city'], - country=venue_details['country'], - name=venue_details['name'], - sr_id=venue_details['sr_id'], - roof_type=venue_details['roof_type'], - state=venue_details['state'], - surface=venue_details['surface'], - zip=venue_details['zip'] - ) - location_embedded = location( - lat=venue_details['lat'], - lng=venue_details['lng'] - ) - venue_info_instance = VenueInfo( - venue1=venue_embedded, - location=location_embedded - ) - mapped_venues[venue_id] = venue_info_instance - #print(len(mapped_venues)) - #print("Mapped_Venues", mapped_venues) - return mapped_venues -def extract_franchise_info(data): - franchise_info_dict = {} - for conference_data in data['conferences']: - for division_data in conference_data['divisions']: - for team_data in division_data['teams']: - franchise_data = team_data['franchise'] - franchise_info = { - 'teamid': team_data['id'], # Corrected field name - 'alias': franchise_data['alias'], - 'id': franchise_data['id'], - 'name': franchise_data['name'] - } - franchise_info_dict[franchise_info['id']] = franchise_info - return franchise_info_dict -def map_franchise_info(franchise_info_dict): - mapped_franchises = {} - for franchise_id, franchise_data in franchise_info_dict.items(): - franchise_embedded = FranchiseInfo( - teamid=franchise_data['teamid'], - id=franchise_data['id'], - alias=franchise_data['alias'], - name=franchise_data['name'] - ) - mapped_franchises[franchise_id] = franchise_embedded - return mapped_franchises -def extract_team_info(data): - team_info_dict = {} - for conference_data in data['conferences']: - for division_data in conference_data['divisions']: - for team_data in division_data['teams']: - team_info = { - 'alias': team_data['alias'], - 'id': team_data['id'], - 'market': team_data['market'], - 'name': team_data['name'], - 'sr_id': team_data['sr_id'], - 'conference_id': conference_data['id'], # Add conference_id - 'division_id': division_data['id'] # Add division_id - } - team_colors_data = team_data.get('team_colors', []) - team_colors = [] - for color_data in team_colors_data: - color_info = { - 'type': color_data['type'], - 'alpha': color_data['alpha'], - 'hex_color': color_data['hex_color'], - 'rgb_color': { - 'red': color_data['rgb_color']['red'], - 'green': color_data['rgb_color']['green'], - 'blue': color_data['rgb_color']['blue'] - } - } - team_colors.append(color_info) - team_info['team_colors'] = team_colors - team_info_dict[team_info['id']] = team_info - return team_info_dict -def map_team_info(team_info_dict): - mapped_teams = {} - for team_id, team_data in team_info_dict.items(): - mapped_team = team( - alias=team_data['alias'], - id=team_data['id'], - market=team_data['market'], - name=team_data['name'], - sr_id=team_data['sr_id'] - ) - mapped_team_colors = [] - for color_data in team_data['team_colors']: - team_color_embedded = team_color( - alpha=color_data['alpha'], - hex_color=color_data['hex_color'], - type=color_data['type'] - ) - mapped_team_colors.append(team_color_embedded) - - mapped_teams[team_id] = TeamInfo( - team=mapped_team, # Removed the list wrapping here - coachs=None, - rgb_colors=None, - team_colors=mapped_team_colors, - conference_id=team_data['conference_id'], - division_id=team_data['division_id'] -) - - return mapped_teams -def extract_league_hierarchy(data): - league_hierarchy_dict = { - 'league': { - 'id': data['league']['id'], - 'name': data['league']['name'], - 'alias': data['league']['alias'] - }, - 'conferences': [] - } - #print("league_hierarchy_dict:", league_hierarchy_dict) - for conference_data in data['conferences']: - conference_info = { - 'id': conference_data['id'], - 'alias': conference_data['alias'], - 'name': conference_data['name'], - 'divisions': [] - } - for division_data in conference_data['divisions']: - division_info = { - 'id': division_data['id'], - 'alias': division_data['alias'], - 'name': division_data['name'], - 'teams': [] - } - for team_data in division_data['teams']: - team_info = { - 'id': team_data['id'], - 'name': team_data['name'], - 'market': team_data['market'], - 'alias': team_data['alias'], - 'sr_id': team_data['sr_id'], - - } # Add other team-related information here - division_info['teams'].append(team_info) - conference_info['divisions'].append(division_info) - league_hierarchy_dict['conferences'].append(conference_info) - #print("Extracted League Hierarchy Data:") - print("league_hierarchy_dict", league_hierarchy_dict) # Print the extracted data - return league_hierarchy_dict -def map_league_hierarchy(league_hierarchy_dict): - print("Type of league_hierarchy_dict:", type(league_hierarchy_dict)) - print("Content of league_hierarchy_dict:", league_hierarchy_dict) - - league_id = league_hierarchy_dict['league']['id'] - current_mapped_league = league( - id=league_id, - alias=league_hierarchy_dict['league']['alias'], - name=league_hierarchy_dict['league']['name'], - conferences=[] - ) - - for conference_data in league_hierarchy_dict['conferences']: - mapped_conference = conference( - cid=conference_data['id'], - calias=conference_data['alias'], - cname=conference_data['name'], - divisions=[] - ) - - mapped_divisions = [] # Initialize a list to hold division objects - - for division_data in conference_data['divisions']: - mapped_teams = [] # Initialize a list to hold team objects - - for team_data in division_data['teams']: - mapped_team = teams( - teamid=team_data['id'], - teamname=team_data['name'] - ) - mapped_teams.append(mapped_team) - - mapped_division = division( - did=division_data['id'], - dalias=division_data['alias'], - dname=division_data['name'], - teams=mapped_teams - ) - - mapped_divisions.append(mapped_division) - - mapped_conference.divisions = mapped_divisions - current_mapped_league.conferences.append(mapped_conference) - - return { - league_id: LeagueHierarchy( - league=current_mapped_league, typeleague=None - ) - } -def save_to_database(mapped_venues, mapped_leaguehierarchy, mapped_franchises, mapped_teams): - print("save_to_database called") - - def update_collection(model_cls, mapped_data, collection_name): - updated_count = 0 - new_count = 0 - for mapped_entry_info in mapped_data: - mapped_entry = mapped_entry_info - - # Update ID before converting to mongo representation - if isinstance(mapped_entry, VenueInfo): - mapped_entry.id = mapped_entry.venue1.id - - mapped_entry_id = mapped_entry.id # Extract the ID from the mapped entry - existing_entry = model_cls.objects(id=mapped_entry_id).first() - - print(f"Checking {collection_name} with ID: {mapped_entry_id}") # Print the ID being checked - - if existing_entry: - updated_fields = [] - for field_name in existing_entry._fields.keys(): - existing_value = getattr(existing_entry, field_name) - mapped_value = getattr(mapped_entry, field_name) - if existing_value != mapped_value: - setattr(existing_entry, field_name, mapped_value) - updated_fields.append(field_name) - if updated_fields: - existing_entry.save() - updated_count += 1 - print(f"Updated {collection_name} {mapped_entry_id}: Updated fields: {', '.join(updated_fields)}") - else: - print(f"No updates needed for {collection_name} {mapped_entry_id}") - else: - mongo_representation = mapped_entry.to_mongo() - if '_id' in mongo_representation and model_cls == FranchiseInfo: - mongo_representation['id'] = mongo_representation.pop('_id') - new_entry = model_cls(**mongo_representation) - new_entry.save() - print(f"Added new {collection_name} with id {new_entry.id}") # Print after saving - new_count += 1 - - print(f"Updated {updated_count} {collection_name}s and added {new_count} new {collection_name}s.") - - update_collection(VenueInfo, mapped_venues.values(), "venue") - update_collection(LeagueHierarchy, mapped_leaguehierarchy.values(), "season") - update_collection(FranchiseInfo, mapped_franchises.values(), "franchise") - update_collection(TeamInfo, mapped_teams.values(), "teams") +import sys +from security import safe_requests + +sys.path.append('..') +from flask import Blueprint +import os +if not hasattr(os, 'add_dll_directory'): + def add_dll_directory(path): + pass +from datetime import datetime +from src.models.franchise_info import(FranchiseInfo) +from src.models.venue_info import(venue1, location, VenueInfo) +from src.models.leaguehierarchy import( teams, division, conference, league,LeagueHierarchy) +from src.models.team_info import(team_color, team, TeamInfo) +bp = Blueprint('LeagueHierarchy', __name__) +@bp.route('/LeagueHierarchy', methods=['GET']) +def fetchandsaveLeagueHierarchy(): + #print("LeagueHierarchy") + url = os.getenv('LEAGUE_HIERARCHY_API_URL') + print(datetime.now(), "Requesting URL:", url) + response = safe_requests.get(url) + print("Response status code:", response.status_code) + if response.status_code != 200: + return f"Call Seasons Successfully{response.status_code}" + data = response.json() + venue_info_dict = extract_venue_info(data) + league_hierarchy_dict = extract_league_hierarchy(data) + franchise_info_dict = extract_franchise_info(data) + team_info_dict = extract_team_info(data) + mapped_venues = map_venue_info(venue_info_dict) + mapped_leaguehierarchy = map_league_hierarchy(league_hierarchy_dict) + mapped_franchises = map_franchise_info(franchise_info_dict) + mapped_teams = map_team_info(team_info_dict) + save_to_database(mapped_venues, mapped_leaguehierarchy, mapped_franchises, mapped_teams) + print("League Hierarchy data fetched and saved successfully.") + return "League Hierarchy data fetched and saved successfully." +def extract_venue_info(data): + venue_info_dict = {} + for conference_data in data['conferences']: + for division_data in conference_data['divisions']: + for team_data in division_data['teams']: + venue_info = { + 'id': team_data['venue']['id'], + 'name': team_data['venue']['name'], + 'city': team_data['venue']['city'], + 'state': team_data['venue'].get('state'), + } + venue_info['country'] = team_data['venue'].get('country') + venue_info['zip'] = team_data['venue'].get('zip') + venue_info['address'] = team_data['venue']['address'] + venue_info['capacity'] = team_data['venue']['capacity'] + venue_info['surface'] = team_data['venue']['surface'] + venue_info['roof_type'] = team_data['venue']['roof_type'] + venue_info['sr_id'] = team_data['venue']['sr_id'] + venue_info['lat'] = team_data.get('venue', {}).get('location', {}).get('lat', None) + venue_info['lng'] = team_data.get('venue', {}).get('location', {}).get('lng', None) + venue_info_dict[team_data['venue']['id']] = venue_info + #print("venue_info_dict:", venue_info_dict) + return venue_info_dict +def map_venue_info(venue_info_dict): + mapped_venues = {} + for venue_id, venue_details in venue_info_dict.items(): + venue_embedded = venue1( + id=venue_details['id'], + address=venue_details['address'], + capacity=venue_details['capacity'], + city=venue_details['city'], + country=venue_details['country'], + name=venue_details['name'], + sr_id=venue_details['sr_id'], + roof_type=venue_details['roof_type'], + state=venue_details['state'], + surface=venue_details['surface'], + zip=venue_details['zip'] + ) + location_embedded = location( + lat=venue_details['lat'], + lng=venue_details['lng'] + ) + venue_info_instance = VenueInfo( + venue1=venue_embedded, + location=location_embedded + ) + mapped_venues[venue_id] = venue_info_instance + #print(len(mapped_venues)) + #print("Mapped_Venues", mapped_venues) + return mapped_venues +def extract_franchise_info(data): + franchise_info_dict = {} + for conference_data in data['conferences']: + for division_data in conference_data['divisions']: + for team_data in division_data['teams']: + franchise_data = team_data['franchise'] + franchise_info = { + 'teamid': team_data['id'], # Corrected field name + 'alias': franchise_data['alias'], + 'id': franchise_data['id'], + 'name': franchise_data['name'] + } + franchise_info_dict[franchise_info['id']] = franchise_info + return franchise_info_dict +def map_franchise_info(franchise_info_dict): + mapped_franchises = {} + for franchise_id, franchise_data in franchise_info_dict.items(): + franchise_embedded = FranchiseInfo( + teamid=franchise_data['teamid'], + id=franchise_data['id'], + alias=franchise_data['alias'], + name=franchise_data['name'] + ) + mapped_franchises[franchise_id] = franchise_embedded + return mapped_franchises +def extract_team_info(data): + team_info_dict = {} + for conference_data in data['conferences']: + for division_data in conference_data['divisions']: + for team_data in division_data['teams']: + team_info = { + 'alias': team_data['alias'], + 'id': team_data['id'], + 'market': team_data['market'], + 'name': team_data['name'], + 'sr_id': team_data['sr_id'], + 'conference_id': conference_data['id'], # Add conference_id + 'division_id': division_data['id'] # Add division_id + } + team_colors_data = team_data.get('team_colors', []) + team_colors = [] + for color_data in team_colors_data: + color_info = { + 'type': color_data['type'], + 'alpha': color_data['alpha'], + 'hex_color': color_data['hex_color'], + 'rgb_color': { + 'red': color_data['rgb_color']['red'], + 'green': color_data['rgb_color']['green'], + 'blue': color_data['rgb_color']['blue'] + } + } + team_colors.append(color_info) + team_info['team_colors'] = team_colors + team_info_dict[team_info['id']] = team_info + return team_info_dict +def map_team_info(team_info_dict): + mapped_teams = {} + for team_id, team_data in team_info_dict.items(): + mapped_team = team( + alias=team_data['alias'], + id=team_data['id'], + market=team_data['market'], + name=team_data['name'], + sr_id=team_data['sr_id'] + ) + mapped_team_colors = [] + for color_data in team_data['team_colors']: + team_color_embedded = team_color( + alpha=color_data['alpha'], + hex_color=color_data['hex_color'], + type=color_data['type'] + ) + mapped_team_colors.append(team_color_embedded) + + mapped_teams[team_id] = TeamInfo( + team=mapped_team, # Removed the list wrapping here + coachs=None, + rgb_colors=None, + team_colors=mapped_team_colors, + conference_id=team_data['conference_id'], + division_id=team_data['division_id'] +) + + return mapped_teams +def extract_league_hierarchy(data): + league_hierarchy_dict = { + 'league': { + 'id': data['league']['id'], + 'name': data['league']['name'], + 'alias': data['league']['alias'] + }, + 'conferences': [] + } + #print("league_hierarchy_dict:", league_hierarchy_dict) + for conference_data in data['conferences']: + conference_info = { + 'id': conference_data['id'], + 'alias': conference_data['alias'], + 'name': conference_data['name'], + 'divisions': [] + } + for division_data in conference_data['divisions']: + division_info = { + 'id': division_data['id'], + 'alias': division_data['alias'], + 'name': division_data['name'], + 'teams': [] + } + for team_data in division_data['teams']: + team_info = { + 'id': team_data['id'], + 'name': team_data['name'], + 'market': team_data['market'], + 'alias': team_data['alias'], + 'sr_id': team_data['sr_id'], + + } # Add other team-related information here + division_info['teams'].append(team_info) + conference_info['divisions'].append(division_info) + league_hierarchy_dict['conferences'].append(conference_info) + #print("Extracted League Hierarchy Data:") + print("league_hierarchy_dict", league_hierarchy_dict) # Print the extracted data + return league_hierarchy_dict +def map_league_hierarchy(league_hierarchy_dict): + print("Type of league_hierarchy_dict:", type(league_hierarchy_dict)) + print("Content of league_hierarchy_dict:", league_hierarchy_dict) + + league_id = league_hierarchy_dict['league']['id'] + current_mapped_league = league( + id=league_id, + alias=league_hierarchy_dict['league']['alias'], + name=league_hierarchy_dict['league']['name'], + conferences=[] + ) + + for conference_data in league_hierarchy_dict['conferences']: + mapped_conference = conference( + cid=conference_data['id'], + calias=conference_data['alias'], + cname=conference_data['name'], + divisions=[] + ) + + mapped_divisions = [] # Initialize a list to hold division objects + + for division_data in conference_data['divisions']: + mapped_teams = [] # Initialize a list to hold team objects + + for team_data in division_data['teams']: + mapped_team = teams( + teamid=team_data['id'], + teamname=team_data['name'] + ) + mapped_teams.append(mapped_team) + + mapped_division = division( + did=division_data['id'], + dalias=division_data['alias'], + dname=division_data['name'], + teams=mapped_teams + ) + + mapped_divisions.append(mapped_division) + + mapped_conference.divisions = mapped_divisions + current_mapped_league.conferences.append(mapped_conference) + + return { + league_id: LeagueHierarchy( + league=current_mapped_league, typeleague=None + ) + } +def save_to_database(mapped_venues, mapped_leaguehierarchy, mapped_franchises, mapped_teams): + print("save_to_database called") + + def update_collection(model_cls, mapped_data, collection_name): + updated_count = 0 + new_count = 0 + for mapped_entry_info in mapped_data: + mapped_entry = mapped_entry_info + + # Update ID before converting to mongo representation + if isinstance(mapped_entry, VenueInfo): + mapped_entry.id = mapped_entry.venue1.id + + mapped_entry_id = mapped_entry.id # Extract the ID from the mapped entry + existing_entry = model_cls.objects(id=mapped_entry_id).first() + + print(f"Checking {collection_name} with ID: {mapped_entry_id}") # Print the ID being checked + + if existing_entry: + updated_fields = [] + for field_name in existing_entry._fields.keys(): + existing_value = getattr(existing_entry, field_name) + mapped_value = getattr(mapped_entry, field_name) + if existing_value != mapped_value: + setattr(existing_entry, field_name, mapped_value) + updated_fields.append(field_name) + if updated_fields: + existing_entry.save() + updated_count += 1 + print(f"Updated {collection_name} {mapped_entry_id}: Updated fields: {', '.join(updated_fields)}") + else: + print(f"No updates needed for {collection_name} {mapped_entry_id}") + else: + mongo_representation = mapped_entry.to_mongo() + if '_id' in mongo_representation and model_cls == FranchiseInfo: + mongo_representation['id'] = mongo_representation.pop('_id') + new_entry = model_cls(**mongo_representation) + new_entry.save() + print(f"Added new {collection_name} with id {new_entry.id}") # Print after saving + new_count += 1 + + print(f"Updated {updated_count} {collection_name}s and added {new_count} new {collection_name}s.") + + update_collection(VenueInfo, mapped_venues.values(), "venue") + update_collection(LeagueHierarchy, mapped_leaguehierarchy.values(), "season") + update_collection(FranchiseInfo, mapped_franchises.values(), "franchise") + update_collection(TeamInfo, mapped_teams.values(), "teams") diff --git a/backend-container/src/apimappings/PBP.py b/backend-container/src/apimappings/PBP.py index 1843f7cb..f5837a02 100644 --- a/backend-container/src/apimappings/PBP.py +++ b/backend-container/src/apimappings/PBP.py @@ -1,31 +1,12 @@ -import base64 -import json -import logging import os -import pickle -import redis -import requests import sys import time -from datetime import datetime, timedelta -from itertools import groupby -from json import JSONEncoder -from logging.handlers import RotatingFileHandler -from operator import itemgetter -from threading import Thread -from apscheduler.schedulers.background import BackgroundScheduler -from bson import Binary, ObjectId -from dateutil.parser import parse -from dotenv import load_dotenv -from flask import Flask, Blueprint, jsonify, make_response, render_template, request, session -from flask_cors import CORS -from flask_mongoengine import MongoEngine -from src.models.play_by_play_info import (quarter, location, possession, start_location, play, score, detail, event, end_situation, start_situation, drive, description, penalty,defensive_team, offensive_team, period, points_after_play, PlayByPlayInfo) -from mongoengine import (connect, DecimalField, Document, EmbeddedDocument, EmbeddedDocumentField, EmbeddedDocumentListField, StringField, UUIDField, IntField, BooleanField, DateTimeField) +from datetime import datetime +from flask import Blueprint +from src.models.play_by_play_info import (quarter, location, possession, start_location, play, score) from pymongo import MongoClient -from pymongo.errors import ConnectionFailure -from uuid import UUID -from waitress import serve +from security import safe_requests + sys.path.append("os.getenv('LPATH')/src/") if not hasattr(os, 'add_dll_directory'): def add_dll_directory(path): @@ -69,7 +50,7 @@ def fetch_and_pbp_by_game(game_id): print("fetch_and_pbp_by_game") API_KEY = os.getenv('APIKEY') PLAY_BY_PLAY_API_URL = f"https://api.sportradar.us/nfl/official/trial/v7/en/games/{game_id}/pbp.json?api_key={api_key}" - response = requests.get(PLAY_BY_PLAY_API_URL, timeout=10) + response = safe_requests.get(PLAY_BY_PLAY_API_URL, timeout=10) print("Response status code:", response.status_code) if response.status_code != 200: return f"GetCurrentSeasonScheduleError for {game_id} {year}: {response.status_code}" @@ -114,7 +95,7 @@ def process_games_for_year(year): def extract_play_by_play_data(source): # Fetch play-by-play data from the source (API, database, file, etc.) # This is an example of extracting from an API - response = requests.get(source) + response = safe_requests.get(source) return response.json() @log_and_catch_exceptions diff --git a/backend-container/src/apimappings/SeasonInfoLists.py b/backend-container/src/apimappings/SeasonInfoLists.py index aa3677cc..4aeaf7ee 100644 --- a/backend-container/src/apimappings/SeasonInfoLists.py +++ b/backend-container/src/apimappings/SeasonInfoLists.py @@ -1,52 +1,54 @@ -import os, time -if not hasattr(os, 'add_dll_directory'): - def add_dll_directory(path): - pass -from mongoengine import connect -from src.models.seasons import(SeasonInfo) -from src.models.team_info import(coach, rgb_color, team, team_color, TeamInfo) -if not hasattr(os, 'add_dll_directory'): - def add_dll_directory(path): - pass -from dotenv import load_dotenv -load_dotenv() -connection_string = os.getenv('MONGODB_URL') -if not connection_string: - raise Exception("MONGO_CONNECTION_STRING must be set") -connect(db='Current_Season', host=connection_string) - -from datetime import datetime -import requests -API_KEY = os.getenv('APIKEY') -URL = "http://api.sportradar.us/nfl/official/trial/v7/en/seasons/{SEASONYEAR}/{SEASONTYPE}/teams/{TeamID}/statistics.json?api_key={API_KEY}" -TEAMID = list({team.id for team in TeamInfo.objects.only("id")}) -SEASONYEAR = list({season.year for season in SeasonInfo.objects.only("year")}) -SEASONTYPE = list( - {season["type"] for season in SeasonInfo.objects.only("type")} -) -print(SEASONYEAR) -print(TEAMID) -print(SEASONTYPE) - -counter = 0 -try: - for team_id in TEAMID: - for year in SEASONYEAR: - for season_type in SEASONTYPE: - constructed_url = URL.format(SEASONYEAR=year, SEASONTYPE=season_type, TeamID=team_id) - print(datetime.now(), "Constructed URL:", constructed_url) - try: - response = requests.get(constructed_url) - if response.status_code != 200: - print(f"Error: API call to {constructed_url} returned status code: {response.status_code}") - except requests.exceptions.RequestException as req_e: - print(f"API request error for {constructed_url}. Error: {req_e}") - counter += 1 - if counter >= 40: - break - if counter >= 40: - break - if counter >= 40: - break -except Exception as e: - print(f"Error: {str(e)}") +import os +from security import safe_requests + +if not hasattr(os, 'add_dll_directory'): + def add_dll_directory(path): + pass +from mongoengine import connect +from src.models.seasons import(SeasonInfo) +from src.models.team_info import(TeamInfo) +if not hasattr(os, 'add_dll_directory'): + def add_dll_directory(path): + pass +from dotenv import load_dotenv +load_dotenv() +connection_string = os.getenv('MONGODB_URL') +if not connection_string: + raise Exception("MONGO_CONNECTION_STRING must be set") +connect(db='Current_Season', host=connection_string) + +from datetime import datetime +import requests +API_KEY = os.getenv('APIKEY') +URL = "http://api.sportradar.us/nfl/official/trial/v7/en/seasons/{SEASONYEAR}/{SEASONTYPE}/teams/{TeamID}/statistics.json?api_key={API_KEY}" +TEAMID = list({team.id for team in TeamInfo.objects.only("id")}) +SEASONYEAR = list({season.year for season in SeasonInfo.objects.only("year")}) +SEASONTYPE = list( + {season["type"] for season in SeasonInfo.objects.only("type")} +) +print(SEASONYEAR) +print(TEAMID) +print(SEASONTYPE) + +counter = 0 +try: + for team_id in TEAMID: + for year in SEASONYEAR: + for season_type in SEASONTYPE: + constructed_url = URL.format(SEASONYEAR=year, SEASONTYPE=season_type, TeamID=team_id) + print(datetime.now(), "Constructed URL:", constructed_url) + try: + response = safe_requests.get(constructed_url) + if response.status_code != 200: + print(f"Error: API call to {constructed_url} returned status code: {response.status_code}") + except requests.exceptions.RequestException as req_e: + print(f"API request error for {constructed_url}. Error: {req_e}") + counter += 1 + if counter >= 40: + break + if counter >= 40: + break + if counter >= 40: + break +except Exception as e: + print(f"Error: {str(e)}") diff --git a/backend-container/src/apimappings/SeasonalStats.py b/backend-container/src/apimappings/SeasonalStats.py index cd496cb9..375bfee6 100644 --- a/backend-container/src/apimappings/SeasonalStats.py +++ b/backend-container/src/apimappings/SeasonalStats.py @@ -1,1127 +1,1113 @@ -import sys -sys.path.append('..') -import requests, pandas, json -from requests.exceptions import RequestException -from flask import Blueprint, jsonify, Flask -import os, time -if not hasattr(os, 'add_dll_directory'): - def add_dll_directory(path): - pass -from pymongo import MongoClient -from datetime import datetime -from uuid import UUID -import json -from mongoengine import connect -from src.models.seasons import(SeasonInfo) -from src.models.player_DCI_info import(player, prospect, primary, position, practice, injury, PlayerDCIinfo) -from src.models.season_stat_oppo_info import(SeasonStatOppo, intreturns, passing, receiving, -defense, receiving, defense, thirddown,fourthdown,goaltogo, redzone, kicks, fieldgoals, -punts, rushing, kickreturns, puntreturns, miscreturns, record, conversions, -kickoffs, fumbles, penalties, touchdowns, interceptions, firstdowns) -from src.models.season_stat_team_info import(SeasonStatTeam, intreturns, passing, -receiving, defense, thirddown,fourthdown,goaltogo, redzone, kicks, fieldgoals, punts, -rushing, kickreturns, puntreturns, record, conversions, kickoffs, fumbles, penalties, -touchdowns, interceptions, firstdowns) -from src.models.season_stat_player_info import(SeasonStatPlayer, intreturns, -passing, receiving, defense, fieldgoals, punts, rushing, extrapoints, -kickreturns, puntreturns, conversions, kickoffs, fumbles, penalties) -from src.models.team_info import(coach, rgb_color, team, team_color, TeamInfo) -from flask_mongoengine import MongoEngine -from mongoengine import (DoesNotExist, DecimalField, EmbeddedDocumentField, Document, - StringField, UUIDField, IntField, BooleanField, DateTimeField, EmbeddedDocument, - EmbeddedDocumentListField) -from bson import ObjectId -import logging -from dotenv import load_dotenv -load_dotenv() - -bp = Blueprint('SeasonalStats', __name__) -@bp.route('/SeasonalStats', methods=['GET']) -def fetch_and_save_seasonal_stats(): - logging.basicConfig(filename='SeasonalStats.log', level=logging.INFO) - logger = logging.getLogger('SeasonalStats') - logger.info("SeasonalStats called") - logging.info("SeasonalStats called") - API_KEY = os.getenv('APIKEY') - URL = "http://api.sportradar.us/nfl/official/trial/v7/en/seasons/{SEASONYEAR}/{SEASONTYPE}/teams/{TEAMID}/statistics.json?api_key={API_KEY}" - TEAMID = list({team.id for team in TeamInfo.objects.only("id")}) - SEASONYEAR = [2023] - #list(set([season.year for season in SeasonInfo.objects.only("year")])) - SEASONTYPE = ['REG'] - #list(set([season["type"] for season in SeasonInfo.objects.only("type")])) - def make_api_call(url): - MAX_RETRIES = 3 - RETRY_DELAY = 10 # wait 10 seconds before retrying in case of a 403 error - retries = 0 - while retries < MAX_RETRIES: - try: - response = requests.get(url) - response.raise_for_status() - return response.json() - except requests.HTTPError as http_err: - if response.status_code == 403: - logger.error(f"API call to {url} returned status code 403. Retrying in {RETRY_DELAY} seconds.") - time.sleep(RETRY_DELAY) - retries += 1 - else: - logger.error(f"HTTP error occurred: {http_err}") - raise - except Exception as err: - logger.error(f"Other error occurred: {err}") - raise - logger.error(f"Max retries reached for URL: {url}. Skipping...") - return None - - total_mapped_seasons = 0 - total_mapped_players = 0 - total_mapped_opposeasonalstats = 0 - total_mapped_player_seasonal_stat = 0 - total_mapped_team_seasonal_stat = 0 - total_mapped_teams = 0 - try: - for team_id in TEAMID: - for year in SEASONYEAR: - for season_type in SEASONTYPE: - constructed_url = URL.format(SEASONYEAR=year, SEASONTYPE=season_type, TEAMID=team_id) - logging.info(f"{datetime.now()} - Requesting URL: {constructed_url}") - try: - response = requests.get(constructed_url) - logging.info(f"Response status code: {response.status_code}") - if response.status_code != 200: - logger.error(f"API call to {constructed_url} returned status code: {response.status_code}") - # Continue to the next iteration if this URL fails - continue - except RequestException as req_e: - logger.error(f"API request error for {constructed_url}. Error: {req_e}") - continue # Continue to the next iteration if there's an error making the request - data = response.json() - logger.info(f"Team Profile Data for TeamID {TEAMID},Year {SEASONYEAR}, SeaosnType {SEASONTYPE} fetched and saved successfully") - # Extract and map season info - mapped_seasons = {} # Initialize an empty dictionary - season_info = extract_season_info(data) - mapped_season = map_season_info(season_info) - mapped_seasons[season_info['id']] = mapped_season - total_mapped_seasons += len(mapped_seasons) - # Extract and map player info - player_info = extract_player_info(data) - mapped_players = map_player_info(player_info) - total_mapped_players += len(mapped_players) - # Extract and map OpponenetSeasonalTeamStats - opponenetseasondata = extract_oppo_seasonal_stats_info(data) - total_mapped_opposeasonalstats += len(opponenetseasondata) - # Extract and map player season stat info - playerseasondata = extract_player_seasonal_stat_info(data) - total_mapped_player_seasonal_stat += len(playerseasondata) - # Extract and map team stat info - teamseasondata = extract_team_seasonal_stat_info(data) - total_mapped_team_seasonal_stat += len(teamseasondata) - # Extract and map team info - team_info_dict = extract_team_info(data) - total_mapped_teams += len(team_info_dict) - # Log keys of data being passed to save_to_database - #logger.info(f"Keys being passed to save_to_database for team_id: {team_id}, year: {year}, season_type: {season_type}:") - #logger.info(f"mapped_seasons keys: {list(mapped_seasons.keys())}") - #logger.info(f"mapped_players keys: {list(mapped_players.keys())}") - #logger.info(f"opponenetseasondata keys: {list(opponenetseasondata.keys())}") - #logger.info(f"teamseasondata keys: {list(teamseasondata.keys())}") - #logger.info(f"playerseasondata keys: {list(playerseasondata.keys())}") - #logger.info(f"team_info_dict keys: {list(team_info_dict.keys())}") - logger.info(f"Keys being passed to save_to_database for team_id: {team_id}, year: {year}, season_type: {season_type}:") - logger.info(f"mapped_seasons keys count: {len(mapped_seasons.keys())}") - logger.info(f"mapped_players keys count: {len(mapped_players.keys())}") - logger.info(f"opponenetseasondata keys count: {len(opponenetseasondata.keys())}") - logger.info(f"teamseasondata keys count: {len(teamseasondata.keys())}") - logger.info(f"playerseasondata keys count: {len(playerseasondata.keys())}") - logger.info(f"team_info_dict keys count: {len(team_info_dict.keys())}") - # Save mapped data to the database - save_to_database(mapped_seasons, mapped_players, opponenetseasondata, teamseasondata, playerseasondata, team_info_dict) - # Add a delay between requests to avoid rate limiting - time.sleep(2) - logger.info(f"Total mapped Seasons: {total_mapped_seasons}") - logger.info(f"Total mapped Players: {total_mapped_players}") - logger.info(f"Total mapped Opponenent Team Season Stats: {total_mapped_opposeasonalstats}") - logger.info(f"Total mapped Player Season Stats: {total_mapped_player_seasonal_stat}") - logger.info(f"Total mapped Team Season Stats: {total_mapped_team_seasonal_stat}") - logger.info(f"Total mapped Teams: {total_mapped_teams}") - return "Team Profile Data Fetched and Saved Successfully" - except Exception as e: - logger.exception("An error occurred:") - return f"Error: {str(e)}", 500 -def extract_season_info(data): - season_data = { - "id": data.get("season", {}).get("id"), - "year": data.get("season", {}).get("year"), - "type": data.get("season", {}).get("type") - } - logging.info("Extracted season data:", season_data) - return season_data -def map_season_info(season_data): - # Print the type and value of season_data for debugging - logging.info(f"Type of season_data: {type(season_data)}") - logging.info("Value of season_data:", season_data) - # Directly mapping the season_data without iterating over its items - season_instance = SeasonInfo( - _id=season_data["id"], - year=season_data["year"], - type=season_data["type"] - ) - # Print the mapped season_instance for debugging - logging.info("Mapped season instance: %s", season_instance) - return season_instance -def extract_player_info(data): - players_data = data.get('players', []) - extracted_players = {} - for player_data in players_data: - player_id = player_data.get("id") # Based on your sample data - if not player_id: - logging.info(f"Debug: Player without ID: {player_data}") - continue - extracted_players[player_id] = { - "fullname": player_data.get("name"), # Changed based on your sample data - "id": player_id, - "jersey": player_data.get("jersey"), - "position": player_data.get("position") - } - logging.info(f"Number of players extracted: {len(extracted_players)}") - return extracted_players -def map_player_info(extracted_players): - mapped_players = {} - for player_id, player_data in extracted_players.items(): - if not player_id: - logging.info(f"Debug: Player data without ID during mapping: {player_data}") - continue - player_instance = player( - fullname=player_data["fullname"], - id=player_data["id"], - jersey=player_data["jersey"], - position=player_data["position"] - ) - player_dci_info_instance = PlayerDCIinfo(playerinfo=player_instance) - mapped_players[player_id] = player_dci_info_instance - logging.info(f"Number of players mapped: {len(mapped_players)}") - return mapped_players -def extract_oppo_seasonal_stats_info(data): - teamid = str(data.get("id")) - seasonid = str(data.get("season", {}).get("id")) - oppo_stats = data.get('opponents', {}) - key = (teamid, seasonid) - opponenetseasondata = {key: {}} - opponenetseasondata_main = opponenetseasondata[key] - opponenetseasondata_main.update({ - "teamid": teamid, - "seasonid": seasonid, - "opponents_played": oppo_stats.get("games_played"), - "touchdowns": { - "passing": oppo_stats.get("touchdowns", {}).get("pass"), - "rush": oppo_stats.get("touchdowns", {}).get("rush"), - "totalreturn": oppo_stats.get("touchdowns", {}).get("total_return"), - "total": oppo_stats.get("touchdowns", {}).get("total"), - "fumblereturn": oppo_stats.get("touchdowns", {}).get("fumble_return"), - "intreturn": oppo_stats.get("touchdowns", {}).get("int_return"), - "kickreturn": oppo_stats.get("touchdowns", {}).get("kick_return"), - "puntreturn": oppo_stats.get("touchdowns", {}).get("punt_return"), - "other": oppo_stats.get("touchdowns", {}).get("other") - }, - "rushing": { - "avgyards": oppo_stats.get("rushing", {}).get("avg_yards"), - "attempts": oppo_stats.get("rushing", {}).get("attempts"), - "touchdowns": oppo_stats.get("rushing", {}).get("touchdowns"), - "tlost": oppo_stats.get("rushing", {}).get("tlost"), - "tlostyards": oppo_stats.get("rushing", {}).get("tlost_yards"), - "yards": oppo_stats.get("rushing", {}).get("yards"), - "longest": oppo_stats.get("rushing", {}).get("longest"), - "longesttouchdown": oppo_stats.get("rushing", {}).get("longest_touchdown"), - "redzoneattempts": oppo_stats.get("rushing", {}).get("redzone_attempts"), - "brokentackles": oppo_stats.get("rushing", {}).get("broken_tackles"), - "kneeldowns": oppo_stats.get("rushing", {}).get("kneel_downs"), - "scrambles": oppo_stats.get("rushing", {}).get("scrambles"), - "yardsaftercontact": oppo_stats.get("rushing", {}).get("yards_after_contact") - }, - "receiving": { - "targets": oppo_stats.get("receiving", {}).get("targets"), - "receptions": oppo_stats.get("receiving", {}).get("receptions"), - "avgyards": oppo_stats.get("receiving", {}).get("avg_yards"), - "yards": oppo_stats.get("receiving", {}).get("yards"), - "touchdowns": oppo_stats.get("receiving", {}).get("touchdowns"), - "yardsaftercatch": oppo_stats.get("receiving", {}).get("yards_after_catch"), - "longest": oppo_stats.get("receiving", {}).get("longest"), - "longesttouchdown": oppo_stats.get("receiving", {}).get("longest_touchdown"), - "redzonetargets": oppo_stats.get("receiving", {}).get("redzone_targets"), - "airyards": oppo_stats.get("receiving", {}).get("air_yards"), - "brokentackles": oppo_stats.get("receiving", {}).get("broken_tackles"), - "droppedpasses": oppo_stats.get("receiving", {}).get("dropped_passes"), - "catchablepasses": oppo_stats.get("receiving", {}).get("catchable_passes"), - "yardsaftercontact": oppo_stats.get("receiving", {}).get("yards_after_contact") - }, - "punts": { - "attempts": oppo_stats.get("punts", {}).get("attempts"), - "yards": oppo_stats.get("punts", {}).get("yards"), - "netyards": oppo_stats.get("punts", {}).get("net_yards"), - "blocked": oppo_stats.get("punts", {}).get("blocked"), - "touchbacks": oppo_stats.get("punts", {}).get("touchbacks"), - "inside20": oppo_stats.get("punts", {}).get("inside_20"), - "returnyards": oppo_stats.get("punts", {}).get("return_yards"), - "avgnetyards": oppo_stats.get("punts", {}).get("avg_net_yards"), - "avgyards": oppo_stats.get("punts", {}).get("avg_yards"), - "longest": oppo_stats.get("punts", {}).get("longest"), - "hangtime": oppo_stats.get("punts", {}).get("hang_time"), - "avghangtime": oppo_stats.get("punts", {}).get("avg_hang_time") - }, - "puntreturns":{ - "avgyards": oppo_stats.get("punt_returns", {}).get("avg_yards"), - "returns": oppo_stats.get("punt_returns", {}).get("returns"), - "yards": oppo_stats.get("punt_returns", {}).get("yards"), - "longest": oppo_stats.get("punt_returns", {}).get("longest"), - "touchdowns": oppo_stats.get("punt_returns", {}).get("touchdowns"), - "longesttouchdown": oppo_stats.get("punt_returns", {}).get("longest_touchdown"), - "faircatches": oppo_stats.get("punt_returns", {}).get("faircatches") - }, - "penalties":{ - "penalties": oppo_stats.get("penalties", {}).get("penalties"), - "yards": oppo_stats.get("penalties", {}).get("yards") - }, - "passing":{ - "attempts": oppo_stats.get("passing", {}).get("attempts"), - "completions": oppo_stats.get("passing", {}).get("completions"), - "comppct": oppo_stats.get("passing", {}).get("cmp_pct"), - "interceptions": oppo_stats.get("passing", {}).get("interceptions"), - "sackyards": oppo_stats.get("passing", {}).get("sack_yards"), - "rating": oppo_stats.get("passing", {}).get("rating"), - "touchdowns": oppo_stats.get("passing", {}).get("touchdowns"), - "avgyards": oppo_stats.get("passing", {}).get("avg_yards"), - "sacks": oppo_stats.get("passing", {}).get("sacks"), - "longest": oppo_stats.get("passing", {}).get("longest"), - "longesttouchdown": oppo_stats.get("passing", {}).get("longest_touchdown"), - "airyards": oppo_stats.get("passing", {}).get("air_yards"), - "redzoneattempts": oppo_stats.get("passing", {}).get("redzone_attempts"), - "netyards": oppo_stats.get("passing", {}).get("net_yards"), - "yards": oppo_stats.get("passing", {}).get("yards"), - "grossyards": oppo_stats.get("passing", {}).get("gross_yards"), - "inttouchdowns": oppo_stats.get("passing", {}).get("int_touchdowns"), - "throwaways": oppo_stats.get("passing", {}).get("throw_aways"), - "poorthrows": oppo_stats.get("passing", {}).get("poor_throws"), - "defendedpasses": oppo_stats.get("passing", {}).get("defended_passes"), - "droppedpasses": oppo_stats.get("passing", {}).get("dropped_passes"), - "spikes": oppo_stats.get("passing", {}).get("spikes"), - "blitzes": oppo_stats.get("passing", {}).get("blitzes"), - "hurries": oppo_stats.get("passing", {}).get("hurries"), - "knockdowns": oppo_stats.get("passing", {}).get("knockdowns"), - "pockettime": oppo_stats.get("passing", {}).get("pocket_time"), - "battedpasses": oppo_stats.get("passing", {}).get("batted_passes"), - "ontargetthrows": oppo_stats.get("passing", {}).get("on_target_throws") - }, - "kickoffs":{ - "endzone": oppo_stats.get("kickoffs", {}).get("endzone"), - "inside20": oppo_stats.get("kickoffs", {}).get("inside_20"), - "returnyards": oppo_stats.get("kickoffs", {}).get("return_yards"), - "returned": oppo_stats.get("kickoffs", {}).get("returned"), - "touchbacks": oppo_stats.get("kickoffs", {}).get("touchbacks"), - "yards": oppo_stats.get("kickoffs", {}).get("yards"), - "outofbounds": oppo_stats.get("kickoffs", {}).get("out_of_bounds"), - "kickoffs": oppo_stats.get("kickoffs", {}).get("kickoffs"), - "onsideattempts": oppo_stats.get("kickoffs", {}).get("onside_attempts"), - "onsidesuccesses": oppo_stats.get("kickoffs", {}).get("onside_successes"), - "squibkicks": oppo_stats.get("kickoffs", {}).get("squib_kicks") - }, - "kickreturns":{ - "avgyards": oppo_stats.get("kick_returns", {}).get("avg_yards"), - "yards": oppo_stats.get("kick_returns", {}).get("yards"), - "longest": oppo_stats.get("kick_returns", {}).get("longest"), - "touchdowns": oppo_stats.get("kick_returns", {}).get("touchdowns"), - "longesttouchdown": oppo_stats.get("kick_returns", {}).get("longest_touchdown"), - "faircatches": oppo_stats.get("kick_returns", {}).get("faircatches"), - "returns": oppo_stats.get("kick_returns", {}).get("returns") - }, - "interceptions":{ - "returnyards": oppo_stats.get("interceptions", {}).get("return_yards"), - "returned": oppo_stats.get("interceptions", {}).get("returned"), - "interceptions": oppo_stats.get("interceptions", {}).get("interceptions"), - }, - "intreturns": { - "avgyards": oppo_stats.get("int_returns", {}).get("avg_yards"), - "yards": oppo_stats.get("int_returns", {}).get("yards"), - "longest": oppo_stats.get("int_returns", {}).get("longest"), - "touchdowns": oppo_stats.get("int_returns", {}).get("touchdowns"), - "longesttouchdown": oppo_stats.get("int_returns", {}).get("longest_touchdown"), - "returns": oppo_stats.get("int_returns", {}).get("returns") - }, - "fumbles": { - "fumbles": oppo_stats.get("fumbles", {}).get("fumbles"), - "lostfumbles": oppo_stats.get("fumbles", {}).get("lost_fumbles"), - "ownrec": oppo_stats.get("fumbles", {}).get("own_rec"), - "ownrecyards": oppo_stats.get("fumbles", {}).get("own_rec_yards"), - "opprec": oppo_stats.get("fumbles", {}).get("opp_rec"), - "opprecyards": oppo_stats.get("fumbles", {}).get("opp_rec_yards"), - "outofbounds": oppo_stats.get("fumbles", {}).get("out_of_bounds"), - "forcedfumbles": oppo_stats.get("fumbles", {}).get("forced_fumbles"), - "ownrectds": oppo_stats.get("fumbles", {}).get("own_rec_tds"), - "opprectds": oppo_stats.get("fumbles", {}).get("opp_rec_tds"), - "ezrectds": oppo_stats.get("fumbles", {}).get("ez_rec_tds") - }, - "firstdowns": { - "passing": oppo_stats.get("first_downs", {}).get("pass"), - "penalty": oppo_stats.get("first_downs", {}).get("penalty"), - "rush": oppo_stats.get("first_downs", {}).get("rush"), - "total": oppo_stats.get("first_downs", {}).get("total") - }, - "fieldgoals": { - "attempts": oppo_stats.get("field_goals", {}).get("attempts"), - "made": oppo_stats.get("field_goals", {}).get("made"), - "blocked": oppo_stats.get("field_goals", {}).get("blocked"), - "yards": oppo_stats.get("field_goals", {}).get("yards"), - "avgyards": oppo_stats.get("field_goals", {}).get("avg_yards"), - "longest": oppo_stats.get("field_goals", {}).get("longest"), - "missed": oppo_stats.get("field_goals", {}).get("missed"), - "pct": oppo_stats.get("field_goals", {}).get("pct"), - "attempts19": oppo_stats.get("field_goals", {}).get("attempts_19"), - "attempts29": oppo_stats.get("field_goals", {}).get("attempts_29"), - "attempts39": oppo_stats.get("field_goals", {}).get("attempts_39"), - "attempts49": oppo_stats.get("field_goals", {}).get("attempts_49"), - "attempts50": oppo_stats.get("field_goals", {}).get("attempts_50"), - "made19": oppo_stats.get("field_goals", {}).get("made_19"), - "made29": oppo_stats.get("field_goals", {}).get("made_29"), - "made39": oppo_stats.get("field_goals", {}).get("made_39"), - "made49": oppo_stats.get("field_goals", {}).get("made_49"), - "made50": oppo_stats.get("field_goals", {}).get("made_50") - }, - "defense": { - "tackles": oppo_stats.get("defense", {}).get("tackles"), - "assists": oppo_stats.get("defense", {}).get("assists"), - "combined": oppo_stats.get("defense", {}).get("combined"), - "sacks": oppo_stats.get("defense", {}).get("sacks"), - "sackyards": oppo_stats.get("defense", {}).get("sack_yards"), - "interceptions": oppo_stats.get("defense", {}).get("interceptions"), - "passesdefended": oppo_stats.get("defense", {}).get("passes_defended"), - "forcedfumbles": oppo_stats.get("defense", {}).get("forced_fumbles"), - "fumblerecoveries": oppo_stats.get("defense", {}).get("fumble_recoveries"), - "qbhits": oppo_stats.get("defense", {}).get("qb_hits"), - "tloss": oppo_stats.get("defense", {}).get("tloss"), - "tlossyards": oppo_stats.get("defense", {}).get("tloss_yards"), - "safeties": oppo_stats.get("defense", {}).get("safeties"), - "sptackles": oppo_stats.get("defense", {}).get("sp_tackles"), - "spassists": oppo_stats.get("defense", {}).get("sp_assists"), - "spforcedfumbles": oppo_stats.get("defense", {}).get("sp_forced_fumbles"), - "spfumblerecoveries": oppo_stats.get("defense", {}).get("sp_fumble_recoveries"), - "spblocks": oppo_stats.get("defense", {}).get("sp_blocks"), - "misctackles": oppo_stats.get("defense", {}).get("misc_tackles"), - "miscassists": oppo_stats.get("defense", {}).get("misc_assists"), - "miscforcedfumbles": oppo_stats.get("defense", {}).get("misc_forced_fumbles"), - "miscfumblerecoveries": oppo_stats.get("defense", {}).get("misc_fumble_recoveries"), - "deftargets": oppo_stats.get("defense", {}).get("def_targets"), - "defcomps": oppo_stats.get("defense", {}).get("def_comps"), - "blitzes": oppo_stats.get("defense", {}).get("blitzes"), - "hurries": oppo_stats.get("defense", {}).get("hurries"), - "knockdowns": oppo_stats.get("defense", {}).get("knockdowns"), - "missedtackles": oppo_stats.get("defense", {}).get("missed_tackles"), - "battedpasses": oppo_stats.get("defense", {}).get("batted_passes"), - "threeandoutsforced": oppo_stats.get("defense", {}).get("three_and_outs_forced"), - "fourthdownstops": oppo_stats.get("defense", {}).get("fourth_down_stops") - }, - "extra_points": { - "conversions": { - "passattempts": oppo_stats.get("extra_points", {}).get("conversions", {}).get("pass_attempts"), - "passsuccesses": oppo_stats.get("extra_points", {}).get("conversions", {}).get("pass_successes"), - "rushattempts": oppo_stats.get("extra_points", {}).get("conversions", {}).get("rush_attempts"), - "rushsuccesses": oppo_stats.get("extra_points", {}).get("conversions", {}).get("rush_successes"), - "defenseattempts": oppo_stats.get("extra_points", {}).get("conversions", {}).get("defense_attempts"), - "defensesuccesses": oppo_stats.get("extra_points", {}).get("conversions", {}).get("defense_successes"), - "turnoversuccesses": oppo_stats.get("extra_points", {}).get("conversions", {}).get("turnover_successes") - }, - "kicks": { - "attempts": oppo_stats.get("extra_points", {}).get("kicks", {}).get("attempts"), - "blocked": oppo_stats.get("extra_points", {}).get("kicks", {}).get("blocked"), - "made": oppo_stats.get("extra_points", {}).get("kicks", {}).get("made"), - "pct": oppo_stats.get("extra_points", {}).get("kicks", {}).get("pct") - }, - }, - "efficiency": { - "goaltogo": { - "attempts": oppo_stats.get("efficiency", {}).get("goaltogo", {}).get("attempts"), - "successes": oppo_stats.get("efficiency", {}).get("goaltogo", {}).get("successes"), - "pct": oppo_stats.get("efficiency", {}).get("goaltogo", {}).get("pct") - }, - "redzone": { - "attempts": oppo_stats.get("efficiency", {}).get("redzone", {}).get("attempts"), - "successes": oppo_stats.get("efficiency", {}).get("redzone", {}).get("successes"), - "pct": oppo_stats.get("efficiency", {}).get("redzone", {}).get("pct") - }, - "thirddown": { - "attempts": oppo_stats.get("efficiency", {}).get("thirddown", {}).get("attempts"), - "successes": oppo_stats.get("efficiency", {}).get("thirddown", {}).get("successes"), - "pct": oppo_stats.get("efficiency", {}).get("thirddown", {}).get("pct") - }, - "fourthdown": { - "attempts": oppo_stats.get("efficiency", {}).get("fourthdown", {}).get("attempts"), - "successes": oppo_stats.get("efficiency", {}).get("fourthdown", {}).get("successes"), - }, - }, - }) - if 'efficiency' in oppo_stats: - efficiencydata = oppo_stats['efficiency'] - for key, value in efficiencydata.items(): - opponenetseasondata_main[key] = value - - if 'extra_points' in oppo_stats: - extrapointsdata = oppo_stats['extra_points'] - for key, value in extrapointsdata.items(): - opponenetseasondata_main[key] = value - #logging.info("Extracted OpponenetData data:", opponenetseasondata) - logging.info(f"Number of OpponenetData extracted: {len(opponenetseasondata)}") - return opponenetseasondata -def extract_team_seasonal_stat_info(data): - teamid = str(data.get("id")) - seasonid = str(data.get("season", {}).get("id")) - team_stats = data.get('record', {}) - key = (teamid, seasonid) - teamseasondata = {key:{}} - teamseasondata_main = teamseasondata[key] - teamseasondata_main.update({ - "teamid": teamid, - "seasonid": seasonid, - "intreturns":{ - "avgyards": team_stats.get("int_returns", {}).get("avg_yards"), - "yards": team_stats.get("int_returns", {}).get("yards"), - "longest": team_stats.get("int_returns", {}).get("longest"), - "touchdowns": team_stats.get("int_returns", {}).get("touchdowns"), - "longesttouchdown": team_stats.get("int_returns", {}).get("longest_touchdown"), - "returns": team_stats.get("int_returns", {}).get("returns") - }, - "passing":{ - "attempts": team_stats.get("passing", {}).get("attempts"), - "completions": team_stats.get("passing", {}).get("completions"), - "cmppct": team_stats.get("passing", {}).get("cmp_pct"), - "interceptions": team_stats.get("passing", {}).get("interceptions"), - "sackyards": team_stats.get("passing", {}).get("sack_yards"), - "rating": team_stats.get("passing", {}).get("rating"), - "touchdowns": team_stats.get("passing", {}).get("touchdowns"), - "avgyards": team_stats.get("passing", {}).get("avg_yards"), - "sacks": team_stats.get("passing", {}).get("sacks"), - "longest": team_stats.get("passing", {}).get("longest"), - "longesttouchdown": team_stats.get("passing", {}).get("longest_touchdown"), - "airyards": team_stats.get("passing", {}).get("air_yards"), - "redzoneattempts": team_stats.get("passing", {}).get("redzone_attempts"), - "netyards": team_stats.get("passing", {}).get("net_yards"), - "yards": team_stats.get("passing", {}).get("yards"), - "grossyards": team_stats.get("passing", {}).get("gross_yards"), - "inttouchdowns": team_stats.get("passing", {}).get("int_touchdowns"), - "throwaways": team_stats.get("passing", {}).get("throw_aways"), - "poorthrows": team_stats.get("passing", {}).get("poor_throws"), - "defendedpasses": team_stats.get("passing", {}).get("defended_passes"), - "droppedpasses": team_stats.get("passing", {}).get("dropped_passes"), - "spikes": team_stats.get("passing", {}).get("spikes"), - "blitzes": team_stats.get("passing", {}).get("blitzes"), - "hurries": team_stats.get("passing", {}).get("hurries"), - "knockdowns": team_stats.get("passing", {}).get("knockdowns"), - "pockettime": team_stats.get("passing", {}).get("pocket_time"), - "battedpasses": team_stats.get("passing", {}).get("batted_passes"), - "ontargetthrows": team_stats.get("passing", {}).get("on_target_throws") - }, - "receiving":{ - "targets": team_stats.get("receiving", {}).get("targets"), - "receptions": team_stats.get("receiving", {}).get("receptions"), - "avgyards": team_stats.get("receiving", {}).get("avg_yards"), - "yards": team_stats.get("receiving", {}).get("yards"), - "touchdowns": team_stats.get("receiving", {}).get("touchdowns"), - "yardsaftercatch": team_stats.get("receiving", {}).get("yards_after_catch"), - "longest": team_stats.get("receiving", {}).get("longest"), - "longesttouchdown": team_stats.get("receiving", {}).get("longest_touchdown"), - "redzonetargets": team_stats.get("receiving", {}).get("redzone_targets"), - "airyards": team_stats.get("receiving", {}).get("air_yards"), - "brokentackles": team_stats.get("receiving", {}).get("broken_tackles"), - "droppedpasses": team_stats.get("receiving", {}).get("dropped_passes"), - "catchablepasses": team_stats.get("receiving", {}).get("catchable_passes"), - "yardsaftercontact": team_stats.get("receiving", {}).get("yards_after_contact") - }, - "defense": { - "tackles": team_stats.get("defense", {}).get("tackles"), - "assists": team_stats.get("defense", {}).get("assists"), - "combined": team_stats.get("defense", {}).get("combined"), - "sacks": team_stats.get("defense", {}).get("sacks"), - "sackyards": team_stats.get("defense", {}).get("sack_yards"), - "interceptions": team_stats.get("defense", {}).get("interceptions"), - "passesdefended": team_stats.get("defense", {}).get("passes_defended"), - "forcedfumbles": team_stats.get("defense", {}).get("forced_fumbles"), - "fumblerecoveries": team_stats.get("defense", {}).get("fumble_recoveries"), - "qbhits": team_stats.get("defense", {}).get("qb_hits"), - "tloss": team_stats.get("defense", {}).get("tloss"), - "tlossyards": team_stats.get("defense", {}).get("tloss_yards"), - "safeties": team_stats.get("defense", {}).get("safeties"), - "sptackles": team_stats.get("defense", {}).get("sp_tackles"), - "spassists": team_stats.get("defense", {}).get("sp_assists"), - "spblocks": team_stats.get("defense", {}).get("sp_blocks"), - "spforcedfumbles": team_stats.get("defense",{}).get("sp_forced_fumbles"), - "spfumblerecoveries": team_stats.get("defense",{}).get("sp_fumble_recoveries"), - "misctackles": team_stats.get("defense",{}).get("misc_tackles"), - "miscassists": team_stats.get("defense",{}).get("misc_assists"), - "miscforcedfumbles": team_stats.get("defense",{}).get("misc_forced_fumbles"), - "miscfumblerecoveries": team_stats.get("defense",{}).get("misc_fumble_recoveries"), - "deftargets": team_stats.get("defense", {}).get("def_targets"), - "defcomps": team_stats.get("defense", {}).get("def_comps"), - "blitzes": team_stats.get("defense", {}).get("blitzes"), - "hurries": team_stats.get("defense", {}).get("hurries"), - "knockdowns": team_stats.get("defense", {}).get("knockdowns"), - "missedtackles": team_stats.get("defense", {}).get("missed_tackles"), - "battedpasses": team_stats.get("defense", {}).get("batted_passes"), - "threeandoutsforced": team_stats.get("defense", {}).get("three_and_outs_forced"), - "fourthdownstops": team_stats.get("defense", {}).get("fourth_down_stops") - }, - "efficiency": { - "goaltogo": { - "attempts": team_stats.get("efficiency", {}).get("goaltogo", {}).get("attempts"), - "successes": team_stats.get("efficiency", {}).get("goaltogo", {}).get("successes"), - "pct": team_stats.get("efficiency", {}).get("goaltogo", {}).get("pct") - }, - "redzone": { - "attempts": team_stats.get("efficiency", {}).get("redzone", {}).get("attempts"), - "successes": team_stats.get("efficiency", {}).get("redzone", {}).get("successes"), - "pct": team_stats.get("efficiency", {}).get("redzone", {}).get("pct") - }, - "thirddown": { - "attempts": team_stats.get("efficiency", {}).get("thirddown", {}).get("attempts"), - "successes": team_stats.get("efficiency", {}).get("thirddown", {}).get("successes"), - "pct": team_stats.get("efficiency", {}).get("thirddown", {}).get("pct") - }, - "fourthdown": { - "attempts": team_stats.get("efficiency", {}).get("fourthdown", {}).get("attempts"), - "successes": team_stats.get("efficiency", {}).get("fourthdown", {}).get("successes"), - "pct": team_stats.get("efficiency", {}).get("fourthdown", {}).get("pct") - }, - }, - "fieldgoals": { - "attempts": team_stats.get("field_goals", {}).get("attempts"), - "made": team_stats.get("field_goals", {}).get("made"), - "blocked": team_stats.get("field_goals", {}).get("blocked"), - "yards": team_stats.get("field_goals", {}).get("yards"), - "avgyards": team_stats.get("field_goals", {}).get("avg_yards"), - "longest": team_stats.get("field_goals", {}).get("longest"), - "missed": team_stats.get("field_goals", {}).get("missed"), - "pct": team_stats.get("field_goals", {}).get("pct"), - "attempts19": team_stats.get("field_goals", {}).get("attempts_19"), - "attempts29": team_stats.get("field_goals", {}).get("attempts_29"), - "attempts39": team_stats.get("field_goals", {}).get("attempts_39"), - "attempts49": team_stats.get("field_goals", {}).get("attempts_49"), - "attempts50": team_stats.get("field_goals", {}).get("attempts_50"), - "made19": team_stats.get("field_goals", {}).get("made_19"), - "made29": team_stats.get("field_goals", {}).get("made_29"), - "made39": team_stats.get("field_goals", {}).get("made_39"), - "made49": team_stats.get("field_goals", {}).get("made_49"), - "made50": team_stats.get("field_goals", {}).get("made_50") - }, - "punts":{ - "attempts": team_stats.get("punts", {}).get("attempts"), - "yards": team_stats.get("punts", {}).get("yards"), - "netyards": team_stats.get("punts", {}).get("net_yards"), - "blocked": team_stats.get("punts", {}).get("blocked"), - "touchbacks": team_stats.get("punts", {}).get("touchbacks"), - "inside20": team_stats.get("punts", {}).get("inside_20"), - "returnyards": team_stats.get("punts", {}).get("return_yards"), - "avgnetyards": team_stats.get("punts", {}).get("avg_net_yards"), - "avgyards": team_stats.get("punts", {}).get("avg_yards"), - "longest": team_stats.get("punts", {}).get("longest"), - "hangtime": team_stats.get("punts", {}).get("hang_time"), - "avghangtime": team_stats.get("punts", {}).get("avg_hang_time") - }, - "rushing": { - "avgyards": team_stats.get("rushing", {}).get("avg_yards"), - "attempts": team_stats.get("rushing", {}).get("attempts"), - "touchdowns": team_stats.get("rushing", {}).get("touchdowns"), - "tlost": team_stats.get("rushing", {}).get("tlost"), - "tlostyards": team_stats.get("rushing", {}).get("tlost_yards"), - "yards": team_stats.get("rushing", {}).get("yards"), - "longest": team_stats.get("rushing", {}).get("longest"), - "longesttouchdown": team_stats.get("rushing", {}).get("longest_touchdown"), - "redzoneattempts": team_stats.get("rushing", {}).get("redzone_attempts"), - "brokentackles": team_stats.get("rushing", {}).get("broken_tackles"), - "kneeldowns": team_stats.get("rushing", {}).get("kneel_downs"), - "scrambles": team_stats.get("rushing", {}).get("scrambles"), - "yardsaftercontact": team_stats.get("rushing", {}).get("yards_after_contact") - }, - "kickreturns":{ - "avgyards": team_stats.get("kick_returns", {}).get("avg_yards"), - "yards": team_stats.get("kick_returns", {}).get("yards"), - "longest": team_stats.get("kick_returns", {}).get("longest"), - "touchdowns": team_stats.get("kick_returns", {}).get("touchdowns"), - "longesttouchdown": team_stats.get("kick_returns", {}).get("longest_touchdown"), - "faircatches": team_stats.get("kick_returns", {}).get("faircatches"), - "returns": team_stats.get("kick_returns", {}).get("returns") - }, - "puntreturns":{ - "avgyards": team_stats.get("punt_returns", {}).get("avg_yards"), - "returns": team_stats.get("punt_returns", {}).get("returns"), - "yards": team_stats.get("punt_returns", {}).get("yards"), - "longest": team_stats.get("punt_returns", {}).get("longest"), - "touchdowns": team_stats.get("punt_returns", {}).get("touchdowns"), - "longesttouchdown": team_stats.get("punt_returns", {}).get("longest_touchdown"), - "faircatches": team_stats.get("punt_returns", {}).get("faircatches") - }, - "miscreturns":{ - "yards": team_stats.get("misc_returns", {}).get("yards"), - "touchdowns": team_stats.get("misc_returns", {}).get("touchdowns"), - "longesttouchdown": team_stats.get("misc_returns", {}).get("longest_touchdown"), - "blkfgtouchdowns": team_stats.get("misc_returns", {}).get("blk_fg_touchdowns"), - "blkpattouchdowns": team_stats.get("misc_returns", {}).get("blk_pat_touchdowns"), - "fgreturntouchdowns": team_stats.get("misc_returns", {}).get("fg_return_touchdowns"), - "ezrectouchdowns": team_stats.get("misc_returns", {}).get("ez_rec_touchdowns"), - "returns": team_stats.get("misc_returns", {}).get("returns") - }, - "games_played": team_stats.get("games_played"), - "extra_points": { - "conversions": { - "passattempts": team_stats.get("extra_points", {}).get("conversions", {}).get("pass_attempts"), - "passsuccesses": team_stats.get("extra_points", {}).get("conversions", {}).get("pass_successes"), - "rushattempts": team_stats.get("extra_points", {}).get("conversions", {}).get("rush_attempts"), - "rushsuccesses": team_stats.get("extra_points", {}).get("conversions", {}).get("rush_successes"), - "defenseattempts": team_stats.get("extra_points", {}).get("conversions", {}).get("defense_attempts"), - "defensesuccesses": team_stats.get("extra_points", {}).get("conversions", {}).get("defense_successes"), - "turnoversuccesses": team_stats.get("extra_points", {}).get("conversions", {}).get("turnover_successes") - }, - "kicks": { - "attempts": team_stats.get("extra_points", {}).get("kicks", {}).get("attempts"), - "blocked": team_stats.get("extra_points", {}).get("kicks", {}).get("blocked"), - "made": team_stats.get("extra_points", {}).get("kicks", {}).get("made"), - "pct": team_stats.get("extra_points", {}).get("kicks", {}).get("pct") - }, - }, - "kickoffs":{ - "endzone": team_stats.get("kickoffs", {}).get("endzone"), - "inside20": team_stats.get("kickoffs", {}).get("inside_20"), - "returnyards": team_stats.get("kickoffs", {}).get("return_yards"), - "returned": team_stats.get("kickoffs", {}).get("returned"), - "touchbacks": team_stats.get("kickoffs", {}).get("touchbacks"), - "yards": team_stats.get("kickoffs", {}).get("yards"), - "outofbounds": team_stats.get("kickoffs", {}).get("out_of_bounds"), - "kickoffs": team_stats.get("kickoffs", {}).get("kickoffs"), - "onsideattempts": team_stats.get("kickoffs", {}).get("onside_attempts"), - "onsidesuccesses": team_stats.get("kickoffs", {}).get("onside_successes"), - "squibkicks": team_stats.get("kickoffs", {}).get("squib_kicks") - }, - "fumbles": { - "fumbles": team_stats.get("fumbles", {}).get("fumbles"), - "lostfumbles": team_stats.get("fumbles", {}).get("lost_fumbles"), - "ownrec": team_stats.get("fumbles", {}).get("own_rec"), - "ownrecyards": team_stats.get("fumbles", {}).get("own_rec_yards"), - "opprec": team_stats.get("fumbles", {}).get("opp_rec"), - "opprecyards": team_stats.get("fumbles", {}).get("opp_rec_yards"), - "outofbounds": team_stats.get("fumbles", {}).get("out_of_bounds"), - "forcedfumbles": team_stats.get("fumbles", {}).get("forced_fumbles"), - "ownrectds": team_stats.get("fumbles", {}).get("own_rec_tds"), - "opprectds": team_stats.get("fumbles", {}).get("opp_rec_tds"), - "ezrectds": team_stats.get("fumbles", {}).get("ez_rec_tds") - }, - "penalties":{ - "penalties": team_stats.get("penalties", {}).get("penalties"), - "yards": team_stats.get("penalties", {}).get("yards") - }, - "touchdowns": { - "passing": team_stats.get("touchdowns", {}).get("pass"), - "rush": team_stats.get("touchdowns", {}).get("rush"), - "totalreturn": team_stats.get("touchdowns", {}).get("total_return"), - "total": team_stats.get("touchdowns", {}).get("total"), - "fumblereturn": team_stats.get("touchdowns", {}).get("fumble_return"), - "intreturn": team_stats.get("touchdowns", {}).get("int_return"), - "kickreturn": team_stats.get("touchdowns", {}).get("kick_return"), - "puntreturn": team_stats.get("touchdowns", {}).get("punt_return"), - "other": team_stats.get("touchdowns", {}).get("other") - }, - "interceptions":{ - "returnyards": team_stats.get("interceptions", {}).get("return_yards"), - "returned": team_stats.get("interceptions", {}).get("returned"), - "interceptions": team_stats.get("interceptions", {}).get("interceptions"), - }, - "firstdowns": { - "passing": team_stats.get("first_downs", {}).get("pass"), - "penalty": team_stats.get("first_downs", {}).get("penalty"), - "rush": team_stats.get("first_downs", {}).get("rush"), - "total": team_stats.get("first_downs", {}).get("total") - }, - }) - if 'efficiency' in team_stats: - efficiencydata = team_stats['efficiency'] - for key, value in efficiencydata.items(): - teamseasondata_main[key] = value - if 'extra_points' in team_stats: - extrapointsdata = team_stats['extra_points'] - for key, value in extrapointsdata.items(): - teamseasondata_main[key] = value - #logging.info("Extracted TEAM SEASON data:", teamseasondata) - logging.info(f"Number of TEAM SEASON extracted: {len(teamseasondata)}") - return teamseasondata -def extract_player_seasonal_stat_info(data): - teamid = str(data.get("id")) - seasonid = str(data.get("season", {}).get("id")) - players_data = data.get("players", []) - # This will be a list to store data for all players - playerseasondata = {} - for player_stats in players_data: - playerid = str(player_stats.get("id")) - key = (playerid, teamid, seasonid) - playerseasondata[key] = ({ - "playerid": playerid, - "teamid": teamid, - "seasonid": seasonid, - "playername": player_stats.get("name"), - "position": player_stats.get("position"), - "jersey": player_stats.get("jersey"), - "gamesplayed": player_stats.get("games_played"), - "gamesstarted": player_stats.get("games_started"), - "intreturns":{ - "longest": player_stats.get("int_returns",{}).get("longest"), - "longesttouchdown": player_stats.get("int_returns",{}).get("longest_touchdown"), - "avgyards": player_stats.get("int_returns",{}).get("avg_yards"), - "returns": player_stats.get("int_returns",{}).get("returns"), - "touchdowns": player_stats.get("int_returns",{}).get("touchdowns"), - "yards": player_stats.get("int_returns",{}).get("yards") - }, - "passing":{ - "attempts": player_stats.get("passing",{}).get("attempts"), - "longest": player_stats.get("passing",{}).get("longest"), - "longesttouchdown": player_stats.get("passing",{}).get("longest_touchdown"), - "airyards": player_stats.get("passing",{}).get("air_yards"), - "avgpockettime": player_stats.get("passing",{}).get("avg_pocket_time"), - "avgyards": player_stats.get("passing",{}).get("avg_yards"), - "battedpasses": player_stats.get("passing",{}).get("batted_passes"), - "blitzes": player_stats.get("passing",{}).get("blitzes"), - "cmppct": player_stats.get("passing",{}).get("cmp_pct"), - "completions": player_stats.get("passing",{}).get("completions"), - "defendedpasses": player_stats.get("passing",{}).get("defended_passes"), - "droppedpasses": player_stats.get("passing",{}).get("dropped_passes"), - "firstdowns": player_stats.get("passing",{}).get("first_downs"), - "grossyards": player_stats.get("passing",{}).get("gross_yards"), - "hurries": player_stats.get("passing",{}).get("hurries"), - "inttouchdowns": player_stats.get("passing",{}).get("int_touchdowns"), - "interceptions": player_stats.get("passing",{}).get("interceptions"), - "knockdowns": player_stats.get("passing",{}).get("knockdowns"), - "netyards": player_stats.get("passing",{}).get("net_yards"), - "ontargetthrows": player_stats.get("passing",{}).get("on_target_throws"), - "pockettime": player_stats.get("passing",{}).get("pocket_time"), - "poorthrows": player_stats.get("passing",{}).get("poor_throws"), - "rating": player_stats.get("passing",{}).get("rating"), - "redzoneattempts": player_stats.get("passing",{}).get("redzone_attempts"), - "sackyards": player_stats.get("passing",{}).get("sack_yards"), - "sacks": player_stats.get("passing",{}).get("sacks"), - "spikes": player_stats.get("passing",{}).get("spikes"), - "throwaways": player_stats.get("passing",{}).get("throw_aways"), - "touchdowns": player_stats.get("passing",{}).get("touchdowns"), - "yards": player_stats.get("passing",{}).get("yards") - }, - "receiving":{ - "longest": player_stats.get("receiving",{}).get("longest"), - "longesttouchdown": player_stats.get("receiving",{}).get("longest_touchdown"), - "airyards": player_stats.get("receiving",{}).get("air_yards"), - "avgyards": player_stats.get("receiving",{}).get("avg_yards"), - "brokentackles": player_stats.get("receiving",{}).get("broken_tackles"), - "catchablepasses": player_stats.get("receiving",{}).get("catchable_passes"), - "droppedpasses": player_stats.get("receiving",{}).get("dropped_passes"), - "firstdowns": player_stats.get("receiving",{}).get("first_downs"), - "receptions": player_stats.get("receiving",{}).get("receptions"), - "redzonetargets": player_stats.get("receiving",{}).get("redzone_targets"), - "targets": player_stats.get("receiving",{}).get("targets"), - "touchdowns": player_stats.get("receiving",{}).get("touchdowns"), - "yards": player_stats.get("receiving",{}).get("yards"), - "yardsaftercatch": player_stats.get("receiving",{}).get("yards_after_catch"), - "yardsaftercontact": player_stats.get("receiving",{}).get("yards_after_contact") - }, - "defense":{ - "assists": player_stats.get("defense",{}).get("assists"), - "battedpasses": player_stats.get("defense",{}).get("batted_passes"), - "blitzes": player_stats.get("defense",{}).get("blitzes"), - "combined": player_stats.get("defense",{}).get("combined"), - "defcomps": player_stats.get("defense",{}).get("def_comps"), - "deftargets": player_stats.get("defense",{}).get("def_targets"), - "forcedfumbles": player_stats.get("defense",{}).get("forced_fumbles"), - "fumblerecoveries": player_stats.get("defense",{}).get("fumble_recoveries"), - "hurries": player_stats.get("defense",{}).get("hurries"), - "interceptions": player_stats.get("defense",{}).get("interceptions"), - "knockdowns": player_stats.get("defense",{}).get("knockdowns"), - "miscassists": player_stats.get("defense",{}).get("misc_assists"), - "miscfumblesforced": player_stats.get("defense",{}).get("misc_forced_fumbles"), - "miscfumblerecoveries": player_stats.get("defense",{}).get("misc_fumble_recoveries"), - "misctackles": player_stats.get("defense",{}).get("misc_tackles"), - "missedtackles": player_stats.get("defense",{}).get("missed_tackles"), - "passesdefended": player_stats.get("defense",{}).get("passes_defended"), - "qbhits": player_stats.get("defense",{}).get("qb_hits"), - "sackyards": player_stats.get("defense",{}).get("sack_yards"), - "sacks": player_stats.get("defense",{}).get("sacks"), - "safeties": player_stats.get("defense",{}).get("safeties"), - "spassists": player_stats.get("defense",{}).get("sp_assists"), - "spblocks": player_stats.get("defense",{}).get("sp_blocks"), - "spforcedfumbles": player_stats.get("defense",{}).get("sp_forced_fumbles"), - "spfumblerecoveries": player_stats.get("defense",{}).get("sp_fumble_recoveries"), - "sptackles": player_stats.get("defense",{}).get("sp_tackles"), - "tackles": player_stats.get("defense",{}).get("tackles"), - "tloss": player_stats.get("defense",{}).get("tloss"), - "tlossyards": player_stats.get("defense",{}).get("tloss_yards") - }, - "fieldgoals":{ - "longest": player_stats.get("field_goals",{}).get("longest"), - "attempts": player_stats.get("field_goals",{}).get("attempts"), - "attempt19": player_stats.get("field_goals",{}).get("attempts_19"), - "attempt29": player_stats.get("field_goals",{}).get("attempts_29"), - "attempt39": player_stats.get("field_goals",{}).get("attempts_39"), - "attempt49": player_stats.get("field_goals",{}).get("attempts_49"), - "attempt50": player_stats.get("field_goals",{}).get("attempts_50"), - "avgyards": player_stats.get("field_goals",{}).get("avg_yards"), - "blocked": player_stats.get("field_goals",{}).get("blocked"), - "made": player_stats.get("field_goals",{}).get("made"), - "made19": player_stats.get("field_goals",{}).get("made_19"), - "made29": player_stats.get("field_goals",{}).get("made_29"), - "made39": player_stats.get("field_goals",{}).get("made_39"), - "made49": player_stats.get("field_goals",{}).get("made_49"), - "made50": player_stats.get("field_goals",{}).get("made_50"), - "missed": player_stats.get("field_goals",{}).get("missed"), - "pct": player_stats.get("field_goals",{}).get("pct"), - "yards": player_stats.get("field_goals",{}).get("yards") - }, - "punts":{ - "longest": player_stats.get("punts",{}).get("longest"), - "attempt": player_stats.get("punts",{}).get("attempts"), - "avghangtime": player_stats.get("punts",{}).get("avg_hang_time"), - "avgnetyards": player_stats.get("punts",{}).get("avg_net_yards"), - "avgyards": player_stats.get("punts",{}).get("avg_yards"), - "blocked": player_stats.get("punts",{}).get("blocked"), - "hangtime": player_stats.get("punts",{}).get("hang_time"), - "inside20": player_stats.get("punts",{}).get("inside_20"), - "netyards": player_stats.get("punts",{}).get("net_yards"), - "returnyards": player_stats.get("punts",{}).get("return_yards"), - "touchbacks": player_stats.get("punts",{}).get("touchbacks"), - "yards": player_stats.get("punts",{}).get("yards") - }, - "rushing":{ - "longest": player_stats.get("rushing",{}).get("longest"), - "longesttouchdown": player_stats.get("rushing",{}).get("longest_touchdown"), - "attempt": player_stats.get("rushing",{}).get("attempts"), - "avgyards": player_stats.get("rushing",{}).get("avg_yards"), - "brokentackles": player_stats.get("rushing",{}).get("broken_tackles"), - "firstdowns": player_stats.get("rushing",{}).get("first_downs"), - "kneeldowns": player_stats.get("rushing",{}).get("kneel_downs"), - "redzoneattempts": player_stats.get("rushing",{}).get("redzone_attempts"), - "scrambles": player_stats.get("rushing",{}).get("scrambles"), - "tlost": player_stats.get("rushing",{}).get("tlost"), - "tlostyards": player_stats.get("rushing",{}).get("tlost_yards"), - "touchdowns": player_stats.get("rushing",{}).get("touchdowns"), - "yards": player_stats.get("rushing",{}).get("yards"), - "yardsaftercontact": player_stats.get("rushing",{}).get("yards_after_contact") - }, - "extra_points":{ - "attempts": player_stats.get("extra_points",{}).get("attempts"), - "blocked": player_stats.get("extra_points",{}).get("blocked"), - "made": player_stats.get("extra_points",{}).get("made"), - "missed": player_stats.get("extra_points",{}).get("missed"), - "pct": player_stats.get("extra_points",{}).get("pct") - }, - "kickreturns":{ - "longest": player_stats.get("kick_returns",{}).get("longest"), - "longesttouchdown": player_stats.get("kick_returns",{}).get("longest_touchdown"), - "avgyards": player_stats.get("kick_returns",{}).get("avg_yards"), - "faircatches": player_stats.get("kick_returns",{}).get("faircatches"), - "returns": player_stats.get("kick_returns",{}).get("returns"), - "touchdowns": player_stats.get("kick_returns",{}).get("touchdowns"), - "yards": player_stats.get("kick_returns",{}).get("yards") - }, - "puntreturns":{ - "longest": player_stats.get("punt_returns",{}).get("longest"), - "longesttouchdown": player_stats.get("punt_returns",{}).get("longest_touchdown"), - "avgyards": player_stats.get("punt_returns",{}).get("avg_yards"), - "faircatches": player_stats.get("punt_returns",{}).get("faircatches"), - "returns": player_stats.get("punt_returns",{}).get("returns"), - "touchdowns": player_stats.get("punt_returns",{}).get("touchdowns"), - "yards": player_stats.get("punt_returns",{}).get("yards") - }, - "conversions":{ - "defenseattempts": player_stats.get("conversions",{}).get("defense_attempts"), - "defensesuccesses": player_stats.get("conversions",{}).get("defense_successes"), - "passattempts": player_stats.get("conversions",{}).get("pass_attempts"), - "passsuccesses": player_stats.get("conversions",{}).get("pass_successes"), - "receiveattempts": player_stats.get("conversions",{}).get("receive_attempts"), - "receivesuccesses": player_stats.get("conversions",{}).get("receive_successes"), - "rushattempts": player_stats.get("conversions",{}).get("rush_attempt"), - "rushsuccesses": player_stats.get("conversions",{}).get("rush_successes") - }, - "kickoffs":{ - "endzone":player_stats.get("kickoffs",{}).get("endzone"), - "inside20":player_stats.get("kickoffs",{}).get("inside_20"), - "kickoffs":player_stats.get("kickoffs",{}).get("kickoffs"), - "onsideattempts":player_stats.get("kickoffs",{}).get("onside_attempts"), - "onsidesuccesses":player_stats.get("kickoffs",{}).get("onside_successes"), - "outofbounds":player_stats.get("kickoffs",{}).get("out_of_bounds"), - "returnyards":player_stats.get("kickoffs",{}).get("return_yards"), - "squibkicks":player_stats.get("kickoffs",{}).get("squib_kicks"), - "touchbacks":player_stats.get("kickoffs",{}).get("touchbacks"), - "yards":player_stats.get("kickoffs",{}).get("yards") - }, - "fumbles": { - "fumbles":player_stats.get("fumbles",{}).get("fumbles"), - "lostfumbles": player_stats.get("fumbles",{}).get("lost_fumbles"), - "ownrec": player_stats.get("fumbles",{}).get("own_rec"), - "ownrecyards": player_stats.get("fumbles",{}).get("own_rec_yards"), - "opprec": player_stats.get("fumbles",{}).get("opp_rec"), - "opprecyards": player_stats.get("fumbles",{}).get("opp_rec_yards"), - "outofbounds": player_stats.get("fumbles",{}).get("out_of_bounds"), - "forcedfumbles": player_stats.get("fumbles",{}).get("forced_fumbles"), - "ownrectds": player_stats.get("fumbles",{}).get("own_rec_tds"), - "opprectds": player_stats.get("fumbles",{}).get("opp_rec_tds"), - "ezrectds": player_stats.get("fumbles",{}).get("ez_rec_tds") - }, - "penalties": { - "penalties": player_stats.get("penalties", {}).get("penalties"), - "yards": player_stats.get("penalties", {}).get("yards"), - "firstdowns": player_stats.get("penalties", {}).get("first_downs") - }, - }) - #logging.info("Extracted PlayerData data:", playerseasondata) - logging.info(f"Number of PlayersData extracted: {len(playerseasondata)}") - return playerseasondata -def extract_team_info(data): - team_id = data.get("id") - team_info = { - "id": team_id, - "name": data.get("name"), - "market": data.get("market"), - "alias": data.get("alias"), - "sr_id": data.get("sr_id"), - } - return {team_id: team_info} -def truncate_dict(d, max_items=5): - """Truncates dictionary to a certain number of items.""" - truncated = dict(list(d.items())[:max_items]) - if len(d) > max_items: - truncated["..."] = "..." - return truncated -def map_dict_to_model(data_dict, model_class): - if not isinstance(data_dict, dict): - logging.info(f"Error: Expected dictionary but received {type(data_dict)} with value {data_dict}") - return None - # Check if the model expects 'id' and map 'playerid' to 'id' accordingly - if 'id' not in data_dict and 'playerid' in data_dict: - data_dict['id'] = data_dict['playerid'] - # Ensure that 'teamid' and 'seasonid' are strings - for key in ['teamid', 'seasonid', 'id']: - if key in data_dict and not isinstance(data_dict[key], str): - logging.info(f"Warning: {key} is not a string. Converting to string.") - data_dict[key] = str(data_dict[key]) - # Handle EmbeddedDocumentField - if isinstance(model_class, EmbeddedDocumentField): - model_instance = model_class.document_type() - else: - model_instance = model_class() - # Print conversion data only for opponentseasondata and teamseasondata - if ( - hasattr(model_instance, 'opponentseasondata') - and 'conversions' in data_dict.get('opponentseasondata', {}) - ): - logging.info(f"Conversions Data for opponentseasondata: {data_dict['opponentseasondata']['conversions']}") - if ( - hasattr(model_instance, 'teamseasondata') - and 'conversions' in data_dict.get('teamseasondata', {}) - ): - logging.info(f"Conversions Data for teamseasondata: {data_dict['teamseasondata']['conversions']}") - for key, value in data_dict.items(): - # Skip the current iteration if the value is None - if value is None: - continue - if hasattr(model_instance, key): # Check if the model instance has the attribute/key - field_type = type(getattr(model_class, key)) - # Handle EmbeddedDocumentListField for lists of embedded documents - if field_type == EmbeddedDocumentListField: - if isinstance(value, list): - nested_model_class = getattr(model_class, key).field.document_type - nested_model_instances = [map_dict_to_model(item, nested_model_class) for item in value] - setattr(model_instance, key, nested_model_instances) - else: - logging.info(f"Warning: Expected a list for {key} but got {type(value)}. Wrapping in a list.") - nested_model_class = getattr(model_class, key).field.document_type - nested_model_instance = map_dict_to_model(value, nested_model_class) - setattr(model_instance, key, [nested_model_instance]) - # Handle EmbeddedDocumentField for embedded documents - elif field_type == EmbeddedDocumentField: - nested_model_class = getattr(model_class, key).document_type - nested_model_instance = map_dict_to_model(value, nested_model_class) - setattr(model_instance, key, nested_model_instance) - # Handle nested dictionaries - elif isinstance(value, dict): - logging.info(f"Mapping nested dictionary for {key}: {value}") - nested_model_class = ( - getattr(model_class, key).document_type - if field_type == EmbeddedDocumentField - else getattr(model_class, key) - ) - nested_model_instance = map_dict_to_model(value, nested_model_class) - setattr(model_instance, key, nested_model_instance) - else: - setattr(model_instance, key, value) - return model_instance -def save_to_database(mapped_seasons, mapped_players, opponenetseasondata, teamseasondata, playerseasondata, team_info_dict): - logging.info("save_to_database called") - if isinstance(opponenetseasondata, tuple): - opponenetseasondata = opponenetseasondata[0] - def handle_non_dict_entry(entry_id, mapped_entry, model_cls): - """ - Handle the non-dictionary entry and possibly convert or format it. - Returns the handled entry which can then be saved to the database. - """ - if isinstance(mapped_entry, model_cls): - return mapped_entry - logging.warning(f"Received unexpected non-dict type {type(mapped_entry)} for {entry_id}. Defaulting to a new model instance.") - return model_cls() - def is_effectively_empty(document): - """Determine if a document is effectively empty""" - for value in document.values(): - if value not in [None, 0, "", []]: - return False - return True - def update_collection(model_cls, mapped_data, collection_name): - logging.info(f"save_to_database called for {collection_name}") - # Adjust for TeamInfo which is a single dictionary, not a dictionary of dictionaries - if collection_name == "TeamInfo": - team_id = mapped_data.get("id") - mapped_data = {team_id: mapped_data} - updated_count = 0 - new_count = 0 - for entry_id, mapped_entry in mapped_data.items(): - if not isinstance(mapped_entry, dict): - mapped_entry = handle_non_dict_entry(entry_id, mapped_entry, model_cls) - new_entry = mapped_entry if isinstance(mapped_entry, model_cls) else map_dict_to_model(mapped_entry, model_cls) - # Specify query condition based on collection_name - if collection_name in ["SeasonStatOppo", "SeasonStatTeam"]: - query_condition = {"seasonid": entry_id[0], "teamid": entry_id[1]} - elif collection_name == "SeasonStatPlayer": - query_condition = {"playerid": entry_id[0], "seasonid": entry_id[1], "teamid": entry_id[2]} - elif collection_name == "SeasonInfo": - query_condition = {"_id": entry_id} - else: - query_condition = {"id": entry_id} - existing_entry = model_cls.objects(**query_condition).first() - if existing_entry: - update_data = {} - for field_name in existing_entry._fields.keys(): - existing_value = getattr(existing_entry, field_name) - mapped_value = getattr(new_entry, field_name, None) - if existing_value != mapped_value and mapped_value is not None: - update_data[field_name] = mapped_value - if update_data: - model_cls.objects(**query_condition).update_one(**{"set__" + key: value for key, value in update_data.items()}) - updated_count += 1 - logging.info(f"Updated {collection_name} {entry_id}: Updated fields: {', '.join(update_data.keys())}") - else: - try: - mongo_representation = new_entry.to_mongo().to_dict() - if '_id' in mongo_representation: - mongo_representation['id'] = mongo_representation.pop('_id') - if is_effectively_empty(mongo_representation): - logging.info(f"Skipping saving an effectively empty document for {collection_name} with id {entry_id}") - continue - new_entry = model_cls(**mongo_representation) - logging.debug(f"Trying to save {collection_name} with data: {mongo_representation}") - new_entry.save() - logging.info(f"Added new {collection_name} with id {new_entry.id}") - new_count += 1 - except Exception as e: - logging.error(f"Error while saving {collection_name} with data: {mongo_representation or 'Failed before mongo representation'}") - raise e - logging.info(f"Updated {updated_count} {collection_name}s and added {new_count} new {collection_name}s.") - update_collection(SeasonInfo, mapped_seasons, "SeasonInfo") - update_collection(PlayerDCIinfo, mapped_players, "PlayerDCIinfo") - update_collection(SeasonStatOppo, opponenetseasondata, "SeasonStatOppo") - update_collection(SeasonStatTeam, teamseasondata, "SeasonStatTeam") - update_collection(SeasonStatPlayer, playerseasondata, "SeasonStatPlayer") - update_collection(TeamInfo, team_info_dict, "TeamInfo") \ No newline at end of file +import sys +from security import safe_requests + +sys.path.append('..') +import requests +from requests.exceptions import RequestException +from flask import Blueprint +import os, time +if not hasattr(os, 'add_dll_directory'): + def add_dll_directory(path): + pass +from datetime import datetime +from src.models.seasons import(SeasonInfo) +from src.models.player_DCI_info import(player, PlayerDCIinfo) +from src.models.season_stat_oppo_info import(SeasonStatOppo) +from src.models.season_stat_team_info import(SeasonStatTeam) +from src.models.season_stat_player_info import(SeasonStatPlayer) +from src.models.team_info import(TeamInfo) +from mongoengine import (EmbeddedDocumentField, EmbeddedDocumentListField) +import logging +from dotenv import load_dotenv +load_dotenv() + +bp = Blueprint('SeasonalStats', __name__) +@bp.route('/SeasonalStats', methods=['GET']) +def fetch_and_save_seasonal_stats(): + logging.basicConfig(filename='SeasonalStats.log', level=logging.INFO) + logger = logging.getLogger('SeasonalStats') + logger.info("SeasonalStats called") + logging.info("SeasonalStats called") + API_KEY = os.getenv('APIKEY') + URL = "http://api.sportradar.us/nfl/official/trial/v7/en/seasons/{SEASONYEAR}/{SEASONTYPE}/teams/{TEAMID}/statistics.json?api_key={API_KEY}" + TEAMID = list({team.id for team in TeamInfo.objects.only("id")}) + SEASONYEAR = [2023] + #list(set([season.year for season in SeasonInfo.objects.only("year")])) + SEASONTYPE = ['REG'] + #list(set([season["type"] for season in SeasonInfo.objects.only("type")])) + def make_api_call(url): + MAX_RETRIES = 3 + RETRY_DELAY = 10 # wait 10 seconds before retrying in case of a 403 error + retries = 0 + while retries < MAX_RETRIES: + try: + response = safe_requests.get(url) + response.raise_for_status() + return response.json() + except requests.HTTPError as http_err: + if response.status_code == 403: + logger.error(f"API call to {url} returned status code 403. Retrying in {RETRY_DELAY} seconds.") + time.sleep(RETRY_DELAY) + retries += 1 + else: + logger.error(f"HTTP error occurred: {http_err}") + raise + except Exception as err: + logger.error(f"Other error occurred: {err}") + raise + logger.error(f"Max retries reached for URL: {url}. Skipping...") + return None + + total_mapped_seasons = 0 + total_mapped_players = 0 + total_mapped_opposeasonalstats = 0 + total_mapped_player_seasonal_stat = 0 + total_mapped_team_seasonal_stat = 0 + total_mapped_teams = 0 + try: + for team_id in TEAMID: + for year in SEASONYEAR: + for season_type in SEASONTYPE: + constructed_url = URL.format(SEASONYEAR=year, SEASONTYPE=season_type, TEAMID=team_id) + logging.info(f"{datetime.now()} - Requesting URL: {constructed_url}") + try: + response = safe_requests.get(constructed_url) + logging.info(f"Response status code: {response.status_code}") + if response.status_code != 200: + logger.error(f"API call to {constructed_url} returned status code: {response.status_code}") + # Continue to the next iteration if this URL fails + continue + except RequestException as req_e: + logger.error(f"API request error for {constructed_url}. Error: {req_e}") + continue # Continue to the next iteration if there's an error making the request + data = response.json() + logger.info(f"Team Profile Data for TeamID {TEAMID},Year {SEASONYEAR}, SeaosnType {SEASONTYPE} fetched and saved successfully") + # Extract and map season info + mapped_seasons = {} # Initialize an empty dictionary + season_info = extract_season_info(data) + mapped_season = map_season_info(season_info) + mapped_seasons[season_info['id']] = mapped_season + total_mapped_seasons += len(mapped_seasons) + # Extract and map player info + player_info = extract_player_info(data) + mapped_players = map_player_info(player_info) + total_mapped_players += len(mapped_players) + # Extract and map OpponenetSeasonalTeamStats + opponenetseasondata = extract_oppo_seasonal_stats_info(data) + total_mapped_opposeasonalstats += len(opponenetseasondata) + # Extract and map player season stat info + playerseasondata = extract_player_seasonal_stat_info(data) + total_mapped_player_seasonal_stat += len(playerseasondata) + # Extract and map team stat info + teamseasondata = extract_team_seasonal_stat_info(data) + total_mapped_team_seasonal_stat += len(teamseasondata) + # Extract and map team info + team_info_dict = extract_team_info(data) + total_mapped_teams += len(team_info_dict) + # Log keys of data being passed to save_to_database + #logger.info(f"Keys being passed to save_to_database for team_id: {team_id}, year: {year}, season_type: {season_type}:") + #logger.info(f"mapped_seasons keys: {list(mapped_seasons.keys())}") + #logger.info(f"mapped_players keys: {list(mapped_players.keys())}") + #logger.info(f"opponenetseasondata keys: {list(opponenetseasondata.keys())}") + #logger.info(f"teamseasondata keys: {list(teamseasondata.keys())}") + #logger.info(f"playerseasondata keys: {list(playerseasondata.keys())}") + #logger.info(f"team_info_dict keys: {list(team_info_dict.keys())}") + logger.info(f"Keys being passed to save_to_database for team_id: {team_id}, year: {year}, season_type: {season_type}:") + logger.info(f"mapped_seasons keys count: {len(mapped_seasons.keys())}") + logger.info(f"mapped_players keys count: {len(mapped_players.keys())}") + logger.info(f"opponenetseasondata keys count: {len(opponenetseasondata.keys())}") + logger.info(f"teamseasondata keys count: {len(teamseasondata.keys())}") + logger.info(f"playerseasondata keys count: {len(playerseasondata.keys())}") + logger.info(f"team_info_dict keys count: {len(team_info_dict.keys())}") + # Save mapped data to the database + save_to_database(mapped_seasons, mapped_players, opponenetseasondata, teamseasondata, playerseasondata, team_info_dict) + # Add a delay between requests to avoid rate limiting + time.sleep(2) + logger.info(f"Total mapped Seasons: {total_mapped_seasons}") + logger.info(f"Total mapped Players: {total_mapped_players}") + logger.info(f"Total mapped Opponenent Team Season Stats: {total_mapped_opposeasonalstats}") + logger.info(f"Total mapped Player Season Stats: {total_mapped_player_seasonal_stat}") + logger.info(f"Total mapped Team Season Stats: {total_mapped_team_seasonal_stat}") + logger.info(f"Total mapped Teams: {total_mapped_teams}") + return "Team Profile Data Fetched and Saved Successfully" + except Exception as e: + logger.exception("An error occurred:") + return f"Error: {str(e)}", 500 +def extract_season_info(data): + season_data = { + "id": data.get("season", {}).get("id"), + "year": data.get("season", {}).get("year"), + "type": data.get("season", {}).get("type") + } + logging.info("Extracted season data:", season_data) + return season_data +def map_season_info(season_data): + # Print the type and value of season_data for debugging + logging.info(f"Type of season_data: {type(season_data)}") + logging.info("Value of season_data:", season_data) + # Directly mapping the season_data without iterating over its items + season_instance = SeasonInfo( + _id=season_data["id"], + year=season_data["year"], + type=season_data["type"] + ) + # Print the mapped season_instance for debugging + logging.info("Mapped season instance: %s", season_instance) + return season_instance +def extract_player_info(data): + players_data = data.get('players', []) + extracted_players = {} + for player_data in players_data: + player_id = player_data.get("id") # Based on your sample data + if not player_id: + logging.info(f"Debug: Player without ID: {player_data}") + continue + extracted_players[player_id] = { + "fullname": player_data.get("name"), # Changed based on your sample data + "id": player_id, + "jersey": player_data.get("jersey"), + "position": player_data.get("position") + } + logging.info(f"Number of players extracted: {len(extracted_players)}") + return extracted_players +def map_player_info(extracted_players): + mapped_players = {} + for player_id, player_data in extracted_players.items(): + if not player_id: + logging.info(f"Debug: Player data without ID during mapping: {player_data}") + continue + player_instance = player( + fullname=player_data["fullname"], + id=player_data["id"], + jersey=player_data["jersey"], + position=player_data["position"] + ) + player_dci_info_instance = PlayerDCIinfo(playerinfo=player_instance) + mapped_players[player_id] = player_dci_info_instance + logging.info(f"Number of players mapped: {len(mapped_players)}") + return mapped_players +def extract_oppo_seasonal_stats_info(data): + teamid = str(data.get("id")) + seasonid = str(data.get("season", {}).get("id")) + oppo_stats = data.get('opponents', {}) + key = (teamid, seasonid) + opponenetseasondata = {key: {}} + opponenetseasondata_main = opponenetseasondata[key] + opponenetseasondata_main.update({ + "teamid": teamid, + "seasonid": seasonid, + "opponents_played": oppo_stats.get("games_played"), + "touchdowns": { + "passing": oppo_stats.get("touchdowns", {}).get("pass"), + "rush": oppo_stats.get("touchdowns", {}).get("rush"), + "totalreturn": oppo_stats.get("touchdowns", {}).get("total_return"), + "total": oppo_stats.get("touchdowns", {}).get("total"), + "fumblereturn": oppo_stats.get("touchdowns", {}).get("fumble_return"), + "intreturn": oppo_stats.get("touchdowns", {}).get("int_return"), + "kickreturn": oppo_stats.get("touchdowns", {}).get("kick_return"), + "puntreturn": oppo_stats.get("touchdowns", {}).get("punt_return"), + "other": oppo_stats.get("touchdowns", {}).get("other") + }, + "rushing": { + "avgyards": oppo_stats.get("rushing", {}).get("avg_yards"), + "attempts": oppo_stats.get("rushing", {}).get("attempts"), + "touchdowns": oppo_stats.get("rushing", {}).get("touchdowns"), + "tlost": oppo_stats.get("rushing", {}).get("tlost"), + "tlostyards": oppo_stats.get("rushing", {}).get("tlost_yards"), + "yards": oppo_stats.get("rushing", {}).get("yards"), + "longest": oppo_stats.get("rushing", {}).get("longest"), + "longesttouchdown": oppo_stats.get("rushing", {}).get("longest_touchdown"), + "redzoneattempts": oppo_stats.get("rushing", {}).get("redzone_attempts"), + "brokentackles": oppo_stats.get("rushing", {}).get("broken_tackles"), + "kneeldowns": oppo_stats.get("rushing", {}).get("kneel_downs"), + "scrambles": oppo_stats.get("rushing", {}).get("scrambles"), + "yardsaftercontact": oppo_stats.get("rushing", {}).get("yards_after_contact") + }, + "receiving": { + "targets": oppo_stats.get("receiving", {}).get("targets"), + "receptions": oppo_stats.get("receiving", {}).get("receptions"), + "avgyards": oppo_stats.get("receiving", {}).get("avg_yards"), + "yards": oppo_stats.get("receiving", {}).get("yards"), + "touchdowns": oppo_stats.get("receiving", {}).get("touchdowns"), + "yardsaftercatch": oppo_stats.get("receiving", {}).get("yards_after_catch"), + "longest": oppo_stats.get("receiving", {}).get("longest"), + "longesttouchdown": oppo_stats.get("receiving", {}).get("longest_touchdown"), + "redzonetargets": oppo_stats.get("receiving", {}).get("redzone_targets"), + "airyards": oppo_stats.get("receiving", {}).get("air_yards"), + "brokentackles": oppo_stats.get("receiving", {}).get("broken_tackles"), + "droppedpasses": oppo_stats.get("receiving", {}).get("dropped_passes"), + "catchablepasses": oppo_stats.get("receiving", {}).get("catchable_passes"), + "yardsaftercontact": oppo_stats.get("receiving", {}).get("yards_after_contact") + }, + "punts": { + "attempts": oppo_stats.get("punts", {}).get("attempts"), + "yards": oppo_stats.get("punts", {}).get("yards"), + "netyards": oppo_stats.get("punts", {}).get("net_yards"), + "blocked": oppo_stats.get("punts", {}).get("blocked"), + "touchbacks": oppo_stats.get("punts", {}).get("touchbacks"), + "inside20": oppo_stats.get("punts", {}).get("inside_20"), + "returnyards": oppo_stats.get("punts", {}).get("return_yards"), + "avgnetyards": oppo_stats.get("punts", {}).get("avg_net_yards"), + "avgyards": oppo_stats.get("punts", {}).get("avg_yards"), + "longest": oppo_stats.get("punts", {}).get("longest"), + "hangtime": oppo_stats.get("punts", {}).get("hang_time"), + "avghangtime": oppo_stats.get("punts", {}).get("avg_hang_time") + }, + "puntreturns":{ + "avgyards": oppo_stats.get("punt_returns", {}).get("avg_yards"), + "returns": oppo_stats.get("punt_returns", {}).get("returns"), + "yards": oppo_stats.get("punt_returns", {}).get("yards"), + "longest": oppo_stats.get("punt_returns", {}).get("longest"), + "touchdowns": oppo_stats.get("punt_returns", {}).get("touchdowns"), + "longesttouchdown": oppo_stats.get("punt_returns", {}).get("longest_touchdown"), + "faircatches": oppo_stats.get("punt_returns", {}).get("faircatches") + }, + "penalties":{ + "penalties": oppo_stats.get("penalties", {}).get("penalties"), + "yards": oppo_stats.get("penalties", {}).get("yards") + }, + "passing":{ + "attempts": oppo_stats.get("passing", {}).get("attempts"), + "completions": oppo_stats.get("passing", {}).get("completions"), + "comppct": oppo_stats.get("passing", {}).get("cmp_pct"), + "interceptions": oppo_stats.get("passing", {}).get("interceptions"), + "sackyards": oppo_stats.get("passing", {}).get("sack_yards"), + "rating": oppo_stats.get("passing", {}).get("rating"), + "touchdowns": oppo_stats.get("passing", {}).get("touchdowns"), + "avgyards": oppo_stats.get("passing", {}).get("avg_yards"), + "sacks": oppo_stats.get("passing", {}).get("sacks"), + "longest": oppo_stats.get("passing", {}).get("longest"), + "longesttouchdown": oppo_stats.get("passing", {}).get("longest_touchdown"), + "airyards": oppo_stats.get("passing", {}).get("air_yards"), + "redzoneattempts": oppo_stats.get("passing", {}).get("redzone_attempts"), + "netyards": oppo_stats.get("passing", {}).get("net_yards"), + "yards": oppo_stats.get("passing", {}).get("yards"), + "grossyards": oppo_stats.get("passing", {}).get("gross_yards"), + "inttouchdowns": oppo_stats.get("passing", {}).get("int_touchdowns"), + "throwaways": oppo_stats.get("passing", {}).get("throw_aways"), + "poorthrows": oppo_stats.get("passing", {}).get("poor_throws"), + "defendedpasses": oppo_stats.get("passing", {}).get("defended_passes"), + "droppedpasses": oppo_stats.get("passing", {}).get("dropped_passes"), + "spikes": oppo_stats.get("passing", {}).get("spikes"), + "blitzes": oppo_stats.get("passing", {}).get("blitzes"), + "hurries": oppo_stats.get("passing", {}).get("hurries"), + "knockdowns": oppo_stats.get("passing", {}).get("knockdowns"), + "pockettime": oppo_stats.get("passing", {}).get("pocket_time"), + "battedpasses": oppo_stats.get("passing", {}).get("batted_passes"), + "ontargetthrows": oppo_stats.get("passing", {}).get("on_target_throws") + }, + "kickoffs":{ + "endzone": oppo_stats.get("kickoffs", {}).get("endzone"), + "inside20": oppo_stats.get("kickoffs", {}).get("inside_20"), + "returnyards": oppo_stats.get("kickoffs", {}).get("return_yards"), + "returned": oppo_stats.get("kickoffs", {}).get("returned"), + "touchbacks": oppo_stats.get("kickoffs", {}).get("touchbacks"), + "yards": oppo_stats.get("kickoffs", {}).get("yards"), + "outofbounds": oppo_stats.get("kickoffs", {}).get("out_of_bounds"), + "kickoffs": oppo_stats.get("kickoffs", {}).get("kickoffs"), + "onsideattempts": oppo_stats.get("kickoffs", {}).get("onside_attempts"), + "onsidesuccesses": oppo_stats.get("kickoffs", {}).get("onside_successes"), + "squibkicks": oppo_stats.get("kickoffs", {}).get("squib_kicks") + }, + "kickreturns":{ + "avgyards": oppo_stats.get("kick_returns", {}).get("avg_yards"), + "yards": oppo_stats.get("kick_returns", {}).get("yards"), + "longest": oppo_stats.get("kick_returns", {}).get("longest"), + "touchdowns": oppo_stats.get("kick_returns", {}).get("touchdowns"), + "longesttouchdown": oppo_stats.get("kick_returns", {}).get("longest_touchdown"), + "faircatches": oppo_stats.get("kick_returns", {}).get("faircatches"), + "returns": oppo_stats.get("kick_returns", {}).get("returns") + }, + "interceptions":{ + "returnyards": oppo_stats.get("interceptions", {}).get("return_yards"), + "returned": oppo_stats.get("interceptions", {}).get("returned"), + "interceptions": oppo_stats.get("interceptions", {}).get("interceptions"), + }, + "intreturns": { + "avgyards": oppo_stats.get("int_returns", {}).get("avg_yards"), + "yards": oppo_stats.get("int_returns", {}).get("yards"), + "longest": oppo_stats.get("int_returns", {}).get("longest"), + "touchdowns": oppo_stats.get("int_returns", {}).get("touchdowns"), + "longesttouchdown": oppo_stats.get("int_returns", {}).get("longest_touchdown"), + "returns": oppo_stats.get("int_returns", {}).get("returns") + }, + "fumbles": { + "fumbles": oppo_stats.get("fumbles", {}).get("fumbles"), + "lostfumbles": oppo_stats.get("fumbles", {}).get("lost_fumbles"), + "ownrec": oppo_stats.get("fumbles", {}).get("own_rec"), + "ownrecyards": oppo_stats.get("fumbles", {}).get("own_rec_yards"), + "opprec": oppo_stats.get("fumbles", {}).get("opp_rec"), + "opprecyards": oppo_stats.get("fumbles", {}).get("opp_rec_yards"), + "outofbounds": oppo_stats.get("fumbles", {}).get("out_of_bounds"), + "forcedfumbles": oppo_stats.get("fumbles", {}).get("forced_fumbles"), + "ownrectds": oppo_stats.get("fumbles", {}).get("own_rec_tds"), + "opprectds": oppo_stats.get("fumbles", {}).get("opp_rec_tds"), + "ezrectds": oppo_stats.get("fumbles", {}).get("ez_rec_tds") + }, + "firstdowns": { + "passing": oppo_stats.get("first_downs", {}).get("pass"), + "penalty": oppo_stats.get("first_downs", {}).get("penalty"), + "rush": oppo_stats.get("first_downs", {}).get("rush"), + "total": oppo_stats.get("first_downs", {}).get("total") + }, + "fieldgoals": { + "attempts": oppo_stats.get("field_goals", {}).get("attempts"), + "made": oppo_stats.get("field_goals", {}).get("made"), + "blocked": oppo_stats.get("field_goals", {}).get("blocked"), + "yards": oppo_stats.get("field_goals", {}).get("yards"), + "avgyards": oppo_stats.get("field_goals", {}).get("avg_yards"), + "longest": oppo_stats.get("field_goals", {}).get("longest"), + "missed": oppo_stats.get("field_goals", {}).get("missed"), + "pct": oppo_stats.get("field_goals", {}).get("pct"), + "attempts19": oppo_stats.get("field_goals", {}).get("attempts_19"), + "attempts29": oppo_stats.get("field_goals", {}).get("attempts_29"), + "attempts39": oppo_stats.get("field_goals", {}).get("attempts_39"), + "attempts49": oppo_stats.get("field_goals", {}).get("attempts_49"), + "attempts50": oppo_stats.get("field_goals", {}).get("attempts_50"), + "made19": oppo_stats.get("field_goals", {}).get("made_19"), + "made29": oppo_stats.get("field_goals", {}).get("made_29"), + "made39": oppo_stats.get("field_goals", {}).get("made_39"), + "made49": oppo_stats.get("field_goals", {}).get("made_49"), + "made50": oppo_stats.get("field_goals", {}).get("made_50") + }, + "defense": { + "tackles": oppo_stats.get("defense", {}).get("tackles"), + "assists": oppo_stats.get("defense", {}).get("assists"), + "combined": oppo_stats.get("defense", {}).get("combined"), + "sacks": oppo_stats.get("defense", {}).get("sacks"), + "sackyards": oppo_stats.get("defense", {}).get("sack_yards"), + "interceptions": oppo_stats.get("defense", {}).get("interceptions"), + "passesdefended": oppo_stats.get("defense", {}).get("passes_defended"), + "forcedfumbles": oppo_stats.get("defense", {}).get("forced_fumbles"), + "fumblerecoveries": oppo_stats.get("defense", {}).get("fumble_recoveries"), + "qbhits": oppo_stats.get("defense", {}).get("qb_hits"), + "tloss": oppo_stats.get("defense", {}).get("tloss"), + "tlossyards": oppo_stats.get("defense", {}).get("tloss_yards"), + "safeties": oppo_stats.get("defense", {}).get("safeties"), + "sptackles": oppo_stats.get("defense", {}).get("sp_tackles"), + "spassists": oppo_stats.get("defense", {}).get("sp_assists"), + "spforcedfumbles": oppo_stats.get("defense", {}).get("sp_forced_fumbles"), + "spfumblerecoveries": oppo_stats.get("defense", {}).get("sp_fumble_recoveries"), + "spblocks": oppo_stats.get("defense", {}).get("sp_blocks"), + "misctackles": oppo_stats.get("defense", {}).get("misc_tackles"), + "miscassists": oppo_stats.get("defense", {}).get("misc_assists"), + "miscforcedfumbles": oppo_stats.get("defense", {}).get("misc_forced_fumbles"), + "miscfumblerecoveries": oppo_stats.get("defense", {}).get("misc_fumble_recoveries"), + "deftargets": oppo_stats.get("defense", {}).get("def_targets"), + "defcomps": oppo_stats.get("defense", {}).get("def_comps"), + "blitzes": oppo_stats.get("defense", {}).get("blitzes"), + "hurries": oppo_stats.get("defense", {}).get("hurries"), + "knockdowns": oppo_stats.get("defense", {}).get("knockdowns"), + "missedtackles": oppo_stats.get("defense", {}).get("missed_tackles"), + "battedpasses": oppo_stats.get("defense", {}).get("batted_passes"), + "threeandoutsforced": oppo_stats.get("defense", {}).get("three_and_outs_forced"), + "fourthdownstops": oppo_stats.get("defense", {}).get("fourth_down_stops") + }, + "extra_points": { + "conversions": { + "passattempts": oppo_stats.get("extra_points", {}).get("conversions", {}).get("pass_attempts"), + "passsuccesses": oppo_stats.get("extra_points", {}).get("conversions", {}).get("pass_successes"), + "rushattempts": oppo_stats.get("extra_points", {}).get("conversions", {}).get("rush_attempts"), + "rushsuccesses": oppo_stats.get("extra_points", {}).get("conversions", {}).get("rush_successes"), + "defenseattempts": oppo_stats.get("extra_points", {}).get("conversions", {}).get("defense_attempts"), + "defensesuccesses": oppo_stats.get("extra_points", {}).get("conversions", {}).get("defense_successes"), + "turnoversuccesses": oppo_stats.get("extra_points", {}).get("conversions", {}).get("turnover_successes") + }, + "kicks": { + "attempts": oppo_stats.get("extra_points", {}).get("kicks", {}).get("attempts"), + "blocked": oppo_stats.get("extra_points", {}).get("kicks", {}).get("blocked"), + "made": oppo_stats.get("extra_points", {}).get("kicks", {}).get("made"), + "pct": oppo_stats.get("extra_points", {}).get("kicks", {}).get("pct") + }, + }, + "efficiency": { + "goaltogo": { + "attempts": oppo_stats.get("efficiency", {}).get("goaltogo", {}).get("attempts"), + "successes": oppo_stats.get("efficiency", {}).get("goaltogo", {}).get("successes"), + "pct": oppo_stats.get("efficiency", {}).get("goaltogo", {}).get("pct") + }, + "redzone": { + "attempts": oppo_stats.get("efficiency", {}).get("redzone", {}).get("attempts"), + "successes": oppo_stats.get("efficiency", {}).get("redzone", {}).get("successes"), + "pct": oppo_stats.get("efficiency", {}).get("redzone", {}).get("pct") + }, + "thirddown": { + "attempts": oppo_stats.get("efficiency", {}).get("thirddown", {}).get("attempts"), + "successes": oppo_stats.get("efficiency", {}).get("thirddown", {}).get("successes"), + "pct": oppo_stats.get("efficiency", {}).get("thirddown", {}).get("pct") + }, + "fourthdown": { + "attempts": oppo_stats.get("efficiency", {}).get("fourthdown", {}).get("attempts"), + "successes": oppo_stats.get("efficiency", {}).get("fourthdown", {}).get("successes"), + }, + }, + }) + if 'efficiency' in oppo_stats: + efficiencydata = oppo_stats['efficiency'] + for key, value in efficiencydata.items(): + opponenetseasondata_main[key] = value + + if 'extra_points' in oppo_stats: + extrapointsdata = oppo_stats['extra_points'] + for key, value in extrapointsdata.items(): + opponenetseasondata_main[key] = value + #logging.info("Extracted OpponenetData data:", opponenetseasondata) + logging.info(f"Number of OpponenetData extracted: {len(opponenetseasondata)}") + return opponenetseasondata +def extract_team_seasonal_stat_info(data): + teamid = str(data.get("id")) + seasonid = str(data.get("season", {}).get("id")) + team_stats = data.get('record', {}) + key = (teamid, seasonid) + teamseasondata = {key:{}} + teamseasondata_main = teamseasondata[key] + teamseasondata_main.update({ + "teamid": teamid, + "seasonid": seasonid, + "intreturns":{ + "avgyards": team_stats.get("int_returns", {}).get("avg_yards"), + "yards": team_stats.get("int_returns", {}).get("yards"), + "longest": team_stats.get("int_returns", {}).get("longest"), + "touchdowns": team_stats.get("int_returns", {}).get("touchdowns"), + "longesttouchdown": team_stats.get("int_returns", {}).get("longest_touchdown"), + "returns": team_stats.get("int_returns", {}).get("returns") + }, + "passing":{ + "attempts": team_stats.get("passing", {}).get("attempts"), + "completions": team_stats.get("passing", {}).get("completions"), + "cmppct": team_stats.get("passing", {}).get("cmp_pct"), + "interceptions": team_stats.get("passing", {}).get("interceptions"), + "sackyards": team_stats.get("passing", {}).get("sack_yards"), + "rating": team_stats.get("passing", {}).get("rating"), + "touchdowns": team_stats.get("passing", {}).get("touchdowns"), + "avgyards": team_stats.get("passing", {}).get("avg_yards"), + "sacks": team_stats.get("passing", {}).get("sacks"), + "longest": team_stats.get("passing", {}).get("longest"), + "longesttouchdown": team_stats.get("passing", {}).get("longest_touchdown"), + "airyards": team_stats.get("passing", {}).get("air_yards"), + "redzoneattempts": team_stats.get("passing", {}).get("redzone_attempts"), + "netyards": team_stats.get("passing", {}).get("net_yards"), + "yards": team_stats.get("passing", {}).get("yards"), + "grossyards": team_stats.get("passing", {}).get("gross_yards"), + "inttouchdowns": team_stats.get("passing", {}).get("int_touchdowns"), + "throwaways": team_stats.get("passing", {}).get("throw_aways"), + "poorthrows": team_stats.get("passing", {}).get("poor_throws"), + "defendedpasses": team_stats.get("passing", {}).get("defended_passes"), + "droppedpasses": team_stats.get("passing", {}).get("dropped_passes"), + "spikes": team_stats.get("passing", {}).get("spikes"), + "blitzes": team_stats.get("passing", {}).get("blitzes"), + "hurries": team_stats.get("passing", {}).get("hurries"), + "knockdowns": team_stats.get("passing", {}).get("knockdowns"), + "pockettime": team_stats.get("passing", {}).get("pocket_time"), + "battedpasses": team_stats.get("passing", {}).get("batted_passes"), + "ontargetthrows": team_stats.get("passing", {}).get("on_target_throws") + }, + "receiving":{ + "targets": team_stats.get("receiving", {}).get("targets"), + "receptions": team_stats.get("receiving", {}).get("receptions"), + "avgyards": team_stats.get("receiving", {}).get("avg_yards"), + "yards": team_stats.get("receiving", {}).get("yards"), + "touchdowns": team_stats.get("receiving", {}).get("touchdowns"), + "yardsaftercatch": team_stats.get("receiving", {}).get("yards_after_catch"), + "longest": team_stats.get("receiving", {}).get("longest"), + "longesttouchdown": team_stats.get("receiving", {}).get("longest_touchdown"), + "redzonetargets": team_stats.get("receiving", {}).get("redzone_targets"), + "airyards": team_stats.get("receiving", {}).get("air_yards"), + "brokentackles": team_stats.get("receiving", {}).get("broken_tackles"), + "droppedpasses": team_stats.get("receiving", {}).get("dropped_passes"), + "catchablepasses": team_stats.get("receiving", {}).get("catchable_passes"), + "yardsaftercontact": team_stats.get("receiving", {}).get("yards_after_contact") + }, + "defense": { + "tackles": team_stats.get("defense", {}).get("tackles"), + "assists": team_stats.get("defense", {}).get("assists"), + "combined": team_stats.get("defense", {}).get("combined"), + "sacks": team_stats.get("defense", {}).get("sacks"), + "sackyards": team_stats.get("defense", {}).get("sack_yards"), + "interceptions": team_stats.get("defense", {}).get("interceptions"), + "passesdefended": team_stats.get("defense", {}).get("passes_defended"), + "forcedfumbles": team_stats.get("defense", {}).get("forced_fumbles"), + "fumblerecoveries": team_stats.get("defense", {}).get("fumble_recoveries"), + "qbhits": team_stats.get("defense", {}).get("qb_hits"), + "tloss": team_stats.get("defense", {}).get("tloss"), + "tlossyards": team_stats.get("defense", {}).get("tloss_yards"), + "safeties": team_stats.get("defense", {}).get("safeties"), + "sptackles": team_stats.get("defense", {}).get("sp_tackles"), + "spassists": team_stats.get("defense", {}).get("sp_assists"), + "spblocks": team_stats.get("defense", {}).get("sp_blocks"), + "spforcedfumbles": team_stats.get("defense",{}).get("sp_forced_fumbles"), + "spfumblerecoveries": team_stats.get("defense",{}).get("sp_fumble_recoveries"), + "misctackles": team_stats.get("defense",{}).get("misc_tackles"), + "miscassists": team_stats.get("defense",{}).get("misc_assists"), + "miscforcedfumbles": team_stats.get("defense",{}).get("misc_forced_fumbles"), + "miscfumblerecoveries": team_stats.get("defense",{}).get("misc_fumble_recoveries"), + "deftargets": team_stats.get("defense", {}).get("def_targets"), + "defcomps": team_stats.get("defense", {}).get("def_comps"), + "blitzes": team_stats.get("defense", {}).get("blitzes"), + "hurries": team_stats.get("defense", {}).get("hurries"), + "knockdowns": team_stats.get("defense", {}).get("knockdowns"), + "missedtackles": team_stats.get("defense", {}).get("missed_tackles"), + "battedpasses": team_stats.get("defense", {}).get("batted_passes"), + "threeandoutsforced": team_stats.get("defense", {}).get("three_and_outs_forced"), + "fourthdownstops": team_stats.get("defense", {}).get("fourth_down_stops") + }, + "efficiency": { + "goaltogo": { + "attempts": team_stats.get("efficiency", {}).get("goaltogo", {}).get("attempts"), + "successes": team_stats.get("efficiency", {}).get("goaltogo", {}).get("successes"), + "pct": team_stats.get("efficiency", {}).get("goaltogo", {}).get("pct") + }, + "redzone": { + "attempts": team_stats.get("efficiency", {}).get("redzone", {}).get("attempts"), + "successes": team_stats.get("efficiency", {}).get("redzone", {}).get("successes"), + "pct": team_stats.get("efficiency", {}).get("redzone", {}).get("pct") + }, + "thirddown": { + "attempts": team_stats.get("efficiency", {}).get("thirddown", {}).get("attempts"), + "successes": team_stats.get("efficiency", {}).get("thirddown", {}).get("successes"), + "pct": team_stats.get("efficiency", {}).get("thirddown", {}).get("pct") + }, + "fourthdown": { + "attempts": team_stats.get("efficiency", {}).get("fourthdown", {}).get("attempts"), + "successes": team_stats.get("efficiency", {}).get("fourthdown", {}).get("successes"), + "pct": team_stats.get("efficiency", {}).get("fourthdown", {}).get("pct") + }, + }, + "fieldgoals": { + "attempts": team_stats.get("field_goals", {}).get("attempts"), + "made": team_stats.get("field_goals", {}).get("made"), + "blocked": team_stats.get("field_goals", {}).get("blocked"), + "yards": team_stats.get("field_goals", {}).get("yards"), + "avgyards": team_stats.get("field_goals", {}).get("avg_yards"), + "longest": team_stats.get("field_goals", {}).get("longest"), + "missed": team_stats.get("field_goals", {}).get("missed"), + "pct": team_stats.get("field_goals", {}).get("pct"), + "attempts19": team_stats.get("field_goals", {}).get("attempts_19"), + "attempts29": team_stats.get("field_goals", {}).get("attempts_29"), + "attempts39": team_stats.get("field_goals", {}).get("attempts_39"), + "attempts49": team_stats.get("field_goals", {}).get("attempts_49"), + "attempts50": team_stats.get("field_goals", {}).get("attempts_50"), + "made19": team_stats.get("field_goals", {}).get("made_19"), + "made29": team_stats.get("field_goals", {}).get("made_29"), + "made39": team_stats.get("field_goals", {}).get("made_39"), + "made49": team_stats.get("field_goals", {}).get("made_49"), + "made50": team_stats.get("field_goals", {}).get("made_50") + }, + "punts":{ + "attempts": team_stats.get("punts", {}).get("attempts"), + "yards": team_stats.get("punts", {}).get("yards"), + "netyards": team_stats.get("punts", {}).get("net_yards"), + "blocked": team_stats.get("punts", {}).get("blocked"), + "touchbacks": team_stats.get("punts", {}).get("touchbacks"), + "inside20": team_stats.get("punts", {}).get("inside_20"), + "returnyards": team_stats.get("punts", {}).get("return_yards"), + "avgnetyards": team_stats.get("punts", {}).get("avg_net_yards"), + "avgyards": team_stats.get("punts", {}).get("avg_yards"), + "longest": team_stats.get("punts", {}).get("longest"), + "hangtime": team_stats.get("punts", {}).get("hang_time"), + "avghangtime": team_stats.get("punts", {}).get("avg_hang_time") + }, + "rushing": { + "avgyards": team_stats.get("rushing", {}).get("avg_yards"), + "attempts": team_stats.get("rushing", {}).get("attempts"), + "touchdowns": team_stats.get("rushing", {}).get("touchdowns"), + "tlost": team_stats.get("rushing", {}).get("tlost"), + "tlostyards": team_stats.get("rushing", {}).get("tlost_yards"), + "yards": team_stats.get("rushing", {}).get("yards"), + "longest": team_stats.get("rushing", {}).get("longest"), + "longesttouchdown": team_stats.get("rushing", {}).get("longest_touchdown"), + "redzoneattempts": team_stats.get("rushing", {}).get("redzone_attempts"), + "brokentackles": team_stats.get("rushing", {}).get("broken_tackles"), + "kneeldowns": team_stats.get("rushing", {}).get("kneel_downs"), + "scrambles": team_stats.get("rushing", {}).get("scrambles"), + "yardsaftercontact": team_stats.get("rushing", {}).get("yards_after_contact") + }, + "kickreturns":{ + "avgyards": team_stats.get("kick_returns", {}).get("avg_yards"), + "yards": team_stats.get("kick_returns", {}).get("yards"), + "longest": team_stats.get("kick_returns", {}).get("longest"), + "touchdowns": team_stats.get("kick_returns", {}).get("touchdowns"), + "longesttouchdown": team_stats.get("kick_returns", {}).get("longest_touchdown"), + "faircatches": team_stats.get("kick_returns", {}).get("faircatches"), + "returns": team_stats.get("kick_returns", {}).get("returns") + }, + "puntreturns":{ + "avgyards": team_stats.get("punt_returns", {}).get("avg_yards"), + "returns": team_stats.get("punt_returns", {}).get("returns"), + "yards": team_stats.get("punt_returns", {}).get("yards"), + "longest": team_stats.get("punt_returns", {}).get("longest"), + "touchdowns": team_stats.get("punt_returns", {}).get("touchdowns"), + "longesttouchdown": team_stats.get("punt_returns", {}).get("longest_touchdown"), + "faircatches": team_stats.get("punt_returns", {}).get("faircatches") + }, + "miscreturns":{ + "yards": team_stats.get("misc_returns", {}).get("yards"), + "touchdowns": team_stats.get("misc_returns", {}).get("touchdowns"), + "longesttouchdown": team_stats.get("misc_returns", {}).get("longest_touchdown"), + "blkfgtouchdowns": team_stats.get("misc_returns", {}).get("blk_fg_touchdowns"), + "blkpattouchdowns": team_stats.get("misc_returns", {}).get("blk_pat_touchdowns"), + "fgreturntouchdowns": team_stats.get("misc_returns", {}).get("fg_return_touchdowns"), + "ezrectouchdowns": team_stats.get("misc_returns", {}).get("ez_rec_touchdowns"), + "returns": team_stats.get("misc_returns", {}).get("returns") + }, + "games_played": team_stats.get("games_played"), + "extra_points": { + "conversions": { + "passattempts": team_stats.get("extra_points", {}).get("conversions", {}).get("pass_attempts"), + "passsuccesses": team_stats.get("extra_points", {}).get("conversions", {}).get("pass_successes"), + "rushattempts": team_stats.get("extra_points", {}).get("conversions", {}).get("rush_attempts"), + "rushsuccesses": team_stats.get("extra_points", {}).get("conversions", {}).get("rush_successes"), + "defenseattempts": team_stats.get("extra_points", {}).get("conversions", {}).get("defense_attempts"), + "defensesuccesses": team_stats.get("extra_points", {}).get("conversions", {}).get("defense_successes"), + "turnoversuccesses": team_stats.get("extra_points", {}).get("conversions", {}).get("turnover_successes") + }, + "kicks": { + "attempts": team_stats.get("extra_points", {}).get("kicks", {}).get("attempts"), + "blocked": team_stats.get("extra_points", {}).get("kicks", {}).get("blocked"), + "made": team_stats.get("extra_points", {}).get("kicks", {}).get("made"), + "pct": team_stats.get("extra_points", {}).get("kicks", {}).get("pct") + }, + }, + "kickoffs":{ + "endzone": team_stats.get("kickoffs", {}).get("endzone"), + "inside20": team_stats.get("kickoffs", {}).get("inside_20"), + "returnyards": team_stats.get("kickoffs", {}).get("return_yards"), + "returned": team_stats.get("kickoffs", {}).get("returned"), + "touchbacks": team_stats.get("kickoffs", {}).get("touchbacks"), + "yards": team_stats.get("kickoffs", {}).get("yards"), + "outofbounds": team_stats.get("kickoffs", {}).get("out_of_bounds"), + "kickoffs": team_stats.get("kickoffs", {}).get("kickoffs"), + "onsideattempts": team_stats.get("kickoffs", {}).get("onside_attempts"), + "onsidesuccesses": team_stats.get("kickoffs", {}).get("onside_successes"), + "squibkicks": team_stats.get("kickoffs", {}).get("squib_kicks") + }, + "fumbles": { + "fumbles": team_stats.get("fumbles", {}).get("fumbles"), + "lostfumbles": team_stats.get("fumbles", {}).get("lost_fumbles"), + "ownrec": team_stats.get("fumbles", {}).get("own_rec"), + "ownrecyards": team_stats.get("fumbles", {}).get("own_rec_yards"), + "opprec": team_stats.get("fumbles", {}).get("opp_rec"), + "opprecyards": team_stats.get("fumbles", {}).get("opp_rec_yards"), + "outofbounds": team_stats.get("fumbles", {}).get("out_of_bounds"), + "forcedfumbles": team_stats.get("fumbles", {}).get("forced_fumbles"), + "ownrectds": team_stats.get("fumbles", {}).get("own_rec_tds"), + "opprectds": team_stats.get("fumbles", {}).get("opp_rec_tds"), + "ezrectds": team_stats.get("fumbles", {}).get("ez_rec_tds") + }, + "penalties":{ + "penalties": team_stats.get("penalties", {}).get("penalties"), + "yards": team_stats.get("penalties", {}).get("yards") + }, + "touchdowns": { + "passing": team_stats.get("touchdowns", {}).get("pass"), + "rush": team_stats.get("touchdowns", {}).get("rush"), + "totalreturn": team_stats.get("touchdowns", {}).get("total_return"), + "total": team_stats.get("touchdowns", {}).get("total"), + "fumblereturn": team_stats.get("touchdowns", {}).get("fumble_return"), + "intreturn": team_stats.get("touchdowns", {}).get("int_return"), + "kickreturn": team_stats.get("touchdowns", {}).get("kick_return"), + "puntreturn": team_stats.get("touchdowns", {}).get("punt_return"), + "other": team_stats.get("touchdowns", {}).get("other") + }, + "interceptions":{ + "returnyards": team_stats.get("interceptions", {}).get("return_yards"), + "returned": team_stats.get("interceptions", {}).get("returned"), + "interceptions": team_stats.get("interceptions", {}).get("interceptions"), + }, + "firstdowns": { + "passing": team_stats.get("first_downs", {}).get("pass"), + "penalty": team_stats.get("first_downs", {}).get("penalty"), + "rush": team_stats.get("first_downs", {}).get("rush"), + "total": team_stats.get("first_downs", {}).get("total") + }, + }) + if 'efficiency' in team_stats: + efficiencydata = team_stats['efficiency'] + for key, value in efficiencydata.items(): + teamseasondata_main[key] = value + if 'extra_points' in team_stats: + extrapointsdata = team_stats['extra_points'] + for key, value in extrapointsdata.items(): + teamseasondata_main[key] = value + #logging.info("Extracted TEAM SEASON data:", teamseasondata) + logging.info(f"Number of TEAM SEASON extracted: {len(teamseasondata)}") + return teamseasondata +def extract_player_seasonal_stat_info(data): + teamid = str(data.get("id")) + seasonid = str(data.get("season", {}).get("id")) + players_data = data.get("players", []) + # This will be a list to store data for all players + playerseasondata = {} + for player_stats in players_data: + playerid = str(player_stats.get("id")) + key = (playerid, teamid, seasonid) + playerseasondata[key] = ({ + "playerid": playerid, + "teamid": teamid, + "seasonid": seasonid, + "playername": player_stats.get("name"), + "position": player_stats.get("position"), + "jersey": player_stats.get("jersey"), + "gamesplayed": player_stats.get("games_played"), + "gamesstarted": player_stats.get("games_started"), + "intreturns":{ + "longest": player_stats.get("int_returns",{}).get("longest"), + "longesttouchdown": player_stats.get("int_returns",{}).get("longest_touchdown"), + "avgyards": player_stats.get("int_returns",{}).get("avg_yards"), + "returns": player_stats.get("int_returns",{}).get("returns"), + "touchdowns": player_stats.get("int_returns",{}).get("touchdowns"), + "yards": player_stats.get("int_returns",{}).get("yards") + }, + "passing":{ + "attempts": player_stats.get("passing",{}).get("attempts"), + "longest": player_stats.get("passing",{}).get("longest"), + "longesttouchdown": player_stats.get("passing",{}).get("longest_touchdown"), + "airyards": player_stats.get("passing",{}).get("air_yards"), + "avgpockettime": player_stats.get("passing",{}).get("avg_pocket_time"), + "avgyards": player_stats.get("passing",{}).get("avg_yards"), + "battedpasses": player_stats.get("passing",{}).get("batted_passes"), + "blitzes": player_stats.get("passing",{}).get("blitzes"), + "cmppct": player_stats.get("passing",{}).get("cmp_pct"), + "completions": player_stats.get("passing",{}).get("completions"), + "defendedpasses": player_stats.get("passing",{}).get("defended_passes"), + "droppedpasses": player_stats.get("passing",{}).get("dropped_passes"), + "firstdowns": player_stats.get("passing",{}).get("first_downs"), + "grossyards": player_stats.get("passing",{}).get("gross_yards"), + "hurries": player_stats.get("passing",{}).get("hurries"), + "inttouchdowns": player_stats.get("passing",{}).get("int_touchdowns"), + "interceptions": player_stats.get("passing",{}).get("interceptions"), + "knockdowns": player_stats.get("passing",{}).get("knockdowns"), + "netyards": player_stats.get("passing",{}).get("net_yards"), + "ontargetthrows": player_stats.get("passing",{}).get("on_target_throws"), + "pockettime": player_stats.get("passing",{}).get("pocket_time"), + "poorthrows": player_stats.get("passing",{}).get("poor_throws"), + "rating": player_stats.get("passing",{}).get("rating"), + "redzoneattempts": player_stats.get("passing",{}).get("redzone_attempts"), + "sackyards": player_stats.get("passing",{}).get("sack_yards"), + "sacks": player_stats.get("passing",{}).get("sacks"), + "spikes": player_stats.get("passing",{}).get("spikes"), + "throwaways": player_stats.get("passing",{}).get("throw_aways"), + "touchdowns": player_stats.get("passing",{}).get("touchdowns"), + "yards": player_stats.get("passing",{}).get("yards") + }, + "receiving":{ + "longest": player_stats.get("receiving",{}).get("longest"), + "longesttouchdown": player_stats.get("receiving",{}).get("longest_touchdown"), + "airyards": player_stats.get("receiving",{}).get("air_yards"), + "avgyards": player_stats.get("receiving",{}).get("avg_yards"), + "brokentackles": player_stats.get("receiving",{}).get("broken_tackles"), + "catchablepasses": player_stats.get("receiving",{}).get("catchable_passes"), + "droppedpasses": player_stats.get("receiving",{}).get("dropped_passes"), + "firstdowns": player_stats.get("receiving",{}).get("first_downs"), + "receptions": player_stats.get("receiving",{}).get("receptions"), + "redzonetargets": player_stats.get("receiving",{}).get("redzone_targets"), + "targets": player_stats.get("receiving",{}).get("targets"), + "touchdowns": player_stats.get("receiving",{}).get("touchdowns"), + "yards": player_stats.get("receiving",{}).get("yards"), + "yardsaftercatch": player_stats.get("receiving",{}).get("yards_after_catch"), + "yardsaftercontact": player_stats.get("receiving",{}).get("yards_after_contact") + }, + "defense":{ + "assists": player_stats.get("defense",{}).get("assists"), + "battedpasses": player_stats.get("defense",{}).get("batted_passes"), + "blitzes": player_stats.get("defense",{}).get("blitzes"), + "combined": player_stats.get("defense",{}).get("combined"), + "defcomps": player_stats.get("defense",{}).get("def_comps"), + "deftargets": player_stats.get("defense",{}).get("def_targets"), + "forcedfumbles": player_stats.get("defense",{}).get("forced_fumbles"), + "fumblerecoveries": player_stats.get("defense",{}).get("fumble_recoveries"), + "hurries": player_stats.get("defense",{}).get("hurries"), + "interceptions": player_stats.get("defense",{}).get("interceptions"), + "knockdowns": player_stats.get("defense",{}).get("knockdowns"), + "miscassists": player_stats.get("defense",{}).get("misc_assists"), + "miscfumblesforced": player_stats.get("defense",{}).get("misc_forced_fumbles"), + "miscfumblerecoveries": player_stats.get("defense",{}).get("misc_fumble_recoveries"), + "misctackles": player_stats.get("defense",{}).get("misc_tackles"), + "missedtackles": player_stats.get("defense",{}).get("missed_tackles"), + "passesdefended": player_stats.get("defense",{}).get("passes_defended"), + "qbhits": player_stats.get("defense",{}).get("qb_hits"), + "sackyards": player_stats.get("defense",{}).get("sack_yards"), + "sacks": player_stats.get("defense",{}).get("sacks"), + "safeties": player_stats.get("defense",{}).get("safeties"), + "spassists": player_stats.get("defense",{}).get("sp_assists"), + "spblocks": player_stats.get("defense",{}).get("sp_blocks"), + "spforcedfumbles": player_stats.get("defense",{}).get("sp_forced_fumbles"), + "spfumblerecoveries": player_stats.get("defense",{}).get("sp_fumble_recoveries"), + "sptackles": player_stats.get("defense",{}).get("sp_tackles"), + "tackles": player_stats.get("defense",{}).get("tackles"), + "tloss": player_stats.get("defense",{}).get("tloss"), + "tlossyards": player_stats.get("defense",{}).get("tloss_yards") + }, + "fieldgoals":{ + "longest": player_stats.get("field_goals",{}).get("longest"), + "attempts": player_stats.get("field_goals",{}).get("attempts"), + "attempt19": player_stats.get("field_goals",{}).get("attempts_19"), + "attempt29": player_stats.get("field_goals",{}).get("attempts_29"), + "attempt39": player_stats.get("field_goals",{}).get("attempts_39"), + "attempt49": player_stats.get("field_goals",{}).get("attempts_49"), + "attempt50": player_stats.get("field_goals",{}).get("attempts_50"), + "avgyards": player_stats.get("field_goals",{}).get("avg_yards"), + "blocked": player_stats.get("field_goals",{}).get("blocked"), + "made": player_stats.get("field_goals",{}).get("made"), + "made19": player_stats.get("field_goals",{}).get("made_19"), + "made29": player_stats.get("field_goals",{}).get("made_29"), + "made39": player_stats.get("field_goals",{}).get("made_39"), + "made49": player_stats.get("field_goals",{}).get("made_49"), + "made50": player_stats.get("field_goals",{}).get("made_50"), + "missed": player_stats.get("field_goals",{}).get("missed"), + "pct": player_stats.get("field_goals",{}).get("pct"), + "yards": player_stats.get("field_goals",{}).get("yards") + }, + "punts":{ + "longest": player_stats.get("punts",{}).get("longest"), + "attempt": player_stats.get("punts",{}).get("attempts"), + "avghangtime": player_stats.get("punts",{}).get("avg_hang_time"), + "avgnetyards": player_stats.get("punts",{}).get("avg_net_yards"), + "avgyards": player_stats.get("punts",{}).get("avg_yards"), + "blocked": player_stats.get("punts",{}).get("blocked"), + "hangtime": player_stats.get("punts",{}).get("hang_time"), + "inside20": player_stats.get("punts",{}).get("inside_20"), + "netyards": player_stats.get("punts",{}).get("net_yards"), + "returnyards": player_stats.get("punts",{}).get("return_yards"), + "touchbacks": player_stats.get("punts",{}).get("touchbacks"), + "yards": player_stats.get("punts",{}).get("yards") + }, + "rushing":{ + "longest": player_stats.get("rushing",{}).get("longest"), + "longesttouchdown": player_stats.get("rushing",{}).get("longest_touchdown"), + "attempt": player_stats.get("rushing",{}).get("attempts"), + "avgyards": player_stats.get("rushing",{}).get("avg_yards"), + "brokentackles": player_stats.get("rushing",{}).get("broken_tackles"), + "firstdowns": player_stats.get("rushing",{}).get("first_downs"), + "kneeldowns": player_stats.get("rushing",{}).get("kneel_downs"), + "redzoneattempts": player_stats.get("rushing",{}).get("redzone_attempts"), + "scrambles": player_stats.get("rushing",{}).get("scrambles"), + "tlost": player_stats.get("rushing",{}).get("tlost"), + "tlostyards": player_stats.get("rushing",{}).get("tlost_yards"), + "touchdowns": player_stats.get("rushing",{}).get("touchdowns"), + "yards": player_stats.get("rushing",{}).get("yards"), + "yardsaftercontact": player_stats.get("rushing",{}).get("yards_after_contact") + }, + "extra_points":{ + "attempts": player_stats.get("extra_points",{}).get("attempts"), + "blocked": player_stats.get("extra_points",{}).get("blocked"), + "made": player_stats.get("extra_points",{}).get("made"), + "missed": player_stats.get("extra_points",{}).get("missed"), + "pct": player_stats.get("extra_points",{}).get("pct") + }, + "kickreturns":{ + "longest": player_stats.get("kick_returns",{}).get("longest"), + "longesttouchdown": player_stats.get("kick_returns",{}).get("longest_touchdown"), + "avgyards": player_stats.get("kick_returns",{}).get("avg_yards"), + "faircatches": player_stats.get("kick_returns",{}).get("faircatches"), + "returns": player_stats.get("kick_returns",{}).get("returns"), + "touchdowns": player_stats.get("kick_returns",{}).get("touchdowns"), + "yards": player_stats.get("kick_returns",{}).get("yards") + }, + "puntreturns":{ + "longest": player_stats.get("punt_returns",{}).get("longest"), + "longesttouchdown": player_stats.get("punt_returns",{}).get("longest_touchdown"), + "avgyards": player_stats.get("punt_returns",{}).get("avg_yards"), + "faircatches": player_stats.get("punt_returns",{}).get("faircatches"), + "returns": player_stats.get("punt_returns",{}).get("returns"), + "touchdowns": player_stats.get("punt_returns",{}).get("touchdowns"), + "yards": player_stats.get("punt_returns",{}).get("yards") + }, + "conversions":{ + "defenseattempts": player_stats.get("conversions",{}).get("defense_attempts"), + "defensesuccesses": player_stats.get("conversions",{}).get("defense_successes"), + "passattempts": player_stats.get("conversions",{}).get("pass_attempts"), + "passsuccesses": player_stats.get("conversions",{}).get("pass_successes"), + "receiveattempts": player_stats.get("conversions",{}).get("receive_attempts"), + "receivesuccesses": player_stats.get("conversions",{}).get("receive_successes"), + "rushattempts": player_stats.get("conversions",{}).get("rush_attempt"), + "rushsuccesses": player_stats.get("conversions",{}).get("rush_successes") + }, + "kickoffs":{ + "endzone":player_stats.get("kickoffs",{}).get("endzone"), + "inside20":player_stats.get("kickoffs",{}).get("inside_20"), + "kickoffs":player_stats.get("kickoffs",{}).get("kickoffs"), + "onsideattempts":player_stats.get("kickoffs",{}).get("onside_attempts"), + "onsidesuccesses":player_stats.get("kickoffs",{}).get("onside_successes"), + "outofbounds":player_stats.get("kickoffs",{}).get("out_of_bounds"), + "returnyards":player_stats.get("kickoffs",{}).get("return_yards"), + "squibkicks":player_stats.get("kickoffs",{}).get("squib_kicks"), + "touchbacks":player_stats.get("kickoffs",{}).get("touchbacks"), + "yards":player_stats.get("kickoffs",{}).get("yards") + }, + "fumbles": { + "fumbles":player_stats.get("fumbles",{}).get("fumbles"), + "lostfumbles": player_stats.get("fumbles",{}).get("lost_fumbles"), + "ownrec": player_stats.get("fumbles",{}).get("own_rec"), + "ownrecyards": player_stats.get("fumbles",{}).get("own_rec_yards"), + "opprec": player_stats.get("fumbles",{}).get("opp_rec"), + "opprecyards": player_stats.get("fumbles",{}).get("opp_rec_yards"), + "outofbounds": player_stats.get("fumbles",{}).get("out_of_bounds"), + "forcedfumbles": player_stats.get("fumbles",{}).get("forced_fumbles"), + "ownrectds": player_stats.get("fumbles",{}).get("own_rec_tds"), + "opprectds": player_stats.get("fumbles",{}).get("opp_rec_tds"), + "ezrectds": player_stats.get("fumbles",{}).get("ez_rec_tds") + }, + "penalties": { + "penalties": player_stats.get("penalties", {}).get("penalties"), + "yards": player_stats.get("penalties", {}).get("yards"), + "firstdowns": player_stats.get("penalties", {}).get("first_downs") + }, + }) + #logging.info("Extracted PlayerData data:", playerseasondata) + logging.info(f"Number of PlayersData extracted: {len(playerseasondata)}") + return playerseasondata +def extract_team_info(data): + team_id = data.get("id") + team_info = { + "id": team_id, + "name": data.get("name"), + "market": data.get("market"), + "alias": data.get("alias"), + "sr_id": data.get("sr_id"), + } + return {team_id: team_info} +def truncate_dict(d, max_items=5): + """Truncates dictionary to a certain number of items.""" + truncated = dict(list(d.items())[:max_items]) + if len(d) > max_items: + truncated["..."] = "..." + return truncated +def map_dict_to_model(data_dict, model_class): + if not isinstance(data_dict, dict): + logging.info(f"Error: Expected dictionary but received {type(data_dict)} with value {data_dict}") + return None + # Check if the model expects 'id' and map 'playerid' to 'id' accordingly + if 'id' not in data_dict and 'playerid' in data_dict: + data_dict['id'] = data_dict['playerid'] + # Ensure that 'teamid' and 'seasonid' are strings + for key in ['teamid', 'seasonid', 'id']: + if key in data_dict and not isinstance(data_dict[key], str): + logging.info(f"Warning: {key} is not a string. Converting to string.") + data_dict[key] = str(data_dict[key]) + # Handle EmbeddedDocumentField + if isinstance(model_class, EmbeddedDocumentField): + model_instance = model_class.document_type() + else: + model_instance = model_class() + # Print conversion data only for opponentseasondata and teamseasondata + if ( + hasattr(model_instance, 'opponentseasondata') + and 'conversions' in data_dict.get('opponentseasondata', {}) + ): + logging.info(f"Conversions Data for opponentseasondata: {data_dict['opponentseasondata']['conversions']}") + if ( + hasattr(model_instance, 'teamseasondata') + and 'conversions' in data_dict.get('teamseasondata', {}) + ): + logging.info(f"Conversions Data for teamseasondata: {data_dict['teamseasondata']['conversions']}") + for key, value in data_dict.items(): + # Skip the current iteration if the value is None + if value is None: + continue + if hasattr(model_instance, key): # Check if the model instance has the attribute/key + field_type = type(getattr(model_class, key)) + # Handle EmbeddedDocumentListField for lists of embedded documents + if field_type == EmbeddedDocumentListField: + if isinstance(value, list): + nested_model_class = getattr(model_class, key).field.document_type + nested_model_instances = [map_dict_to_model(item, nested_model_class) for item in value] + setattr(model_instance, key, nested_model_instances) + else: + logging.info(f"Warning: Expected a list for {key} but got {type(value)}. Wrapping in a list.") + nested_model_class = getattr(model_class, key).field.document_type + nested_model_instance = map_dict_to_model(value, nested_model_class) + setattr(model_instance, key, [nested_model_instance]) + # Handle EmbeddedDocumentField for embedded documents + elif field_type == EmbeddedDocumentField: + nested_model_class = getattr(model_class, key).document_type + nested_model_instance = map_dict_to_model(value, nested_model_class) + setattr(model_instance, key, nested_model_instance) + # Handle nested dictionaries + elif isinstance(value, dict): + logging.info(f"Mapping nested dictionary for {key}: {value}") + nested_model_class = ( + getattr(model_class, key).document_type + if field_type == EmbeddedDocumentField + else getattr(model_class, key) + ) + nested_model_instance = map_dict_to_model(value, nested_model_class) + setattr(model_instance, key, nested_model_instance) + else: + setattr(model_instance, key, value) + return model_instance +def save_to_database(mapped_seasons, mapped_players, opponenetseasondata, teamseasondata, playerseasondata, team_info_dict): + logging.info("save_to_database called") + if isinstance(opponenetseasondata, tuple): + opponenetseasondata = opponenetseasondata[0] + def handle_non_dict_entry(entry_id, mapped_entry, model_cls): + """ + Handle the non-dictionary entry and possibly convert or format it. + Returns the handled entry which can then be saved to the database. + """ + if isinstance(mapped_entry, model_cls): + return mapped_entry + logging.warning(f"Received unexpected non-dict type {type(mapped_entry)} for {entry_id}. Defaulting to a new model instance.") + return model_cls() + def is_effectively_empty(document): + """Determine if a document is effectively empty""" + for value in document.values(): + if value not in [None, 0, "", []]: + return False + return True + def update_collection(model_cls, mapped_data, collection_name): + logging.info(f"save_to_database called for {collection_name}") + # Adjust for TeamInfo which is a single dictionary, not a dictionary of dictionaries + if collection_name == "TeamInfo": + team_id = mapped_data.get("id") + mapped_data = {team_id: mapped_data} + updated_count = 0 + new_count = 0 + for entry_id, mapped_entry in mapped_data.items(): + if not isinstance(mapped_entry, dict): + mapped_entry = handle_non_dict_entry(entry_id, mapped_entry, model_cls) + new_entry = mapped_entry if isinstance(mapped_entry, model_cls) else map_dict_to_model(mapped_entry, model_cls) + # Specify query condition based on collection_name + if collection_name in ["SeasonStatOppo", "SeasonStatTeam"]: + query_condition = {"seasonid": entry_id[0], "teamid": entry_id[1]} + elif collection_name == "SeasonStatPlayer": + query_condition = {"playerid": entry_id[0], "seasonid": entry_id[1], "teamid": entry_id[2]} + elif collection_name == "SeasonInfo": + query_condition = {"_id": entry_id} + else: + query_condition = {"id": entry_id} + existing_entry = model_cls.objects(**query_condition).first() + if existing_entry: + update_data = {} + for field_name in existing_entry._fields.keys(): + existing_value = getattr(existing_entry, field_name) + mapped_value = getattr(new_entry, field_name, None) + if existing_value != mapped_value and mapped_value is not None: + update_data[field_name] = mapped_value + if update_data: + model_cls.objects(**query_condition).update_one(**{"set__" + key: value for key, value in update_data.items()}) + updated_count += 1 + logging.info(f"Updated {collection_name} {entry_id}: Updated fields: {', '.join(update_data.keys())}") + else: + try: + mongo_representation = new_entry.to_mongo().to_dict() + if '_id' in mongo_representation: + mongo_representation['id'] = mongo_representation.pop('_id') + if is_effectively_empty(mongo_representation): + logging.info(f"Skipping saving an effectively empty document for {collection_name} with id {entry_id}") + continue + new_entry = model_cls(**mongo_representation) + logging.debug(f"Trying to save {collection_name} with data: {mongo_representation}") + new_entry.save() + logging.info(f"Added new {collection_name} with id {new_entry.id}") + new_count += 1 + except Exception as e: + logging.error(f"Error while saving {collection_name} with data: {mongo_representation or 'Failed before mongo representation'}") + raise e + logging.info(f"Updated {updated_count} {collection_name}s and added {new_count} new {collection_name}s.") + update_collection(SeasonInfo, mapped_seasons, "SeasonInfo") + update_collection(PlayerDCIinfo, mapped_players, "PlayerDCIinfo") + update_collection(SeasonStatOppo, opponenetseasondata, "SeasonStatOppo") + update_collection(SeasonStatTeam, teamseasondata, "SeasonStatTeam") + update_collection(SeasonStatPlayer, playerseasondata, "SeasonStatPlayer") + update_collection(TeamInfo, team_info_dict, "TeamInfo") diff --git a/backend-container/src/apimappings/Seasons.py b/backend-container/src/apimappings/Seasons.py index 110db05d..9f860136 100644 --- a/backend-container/src/apimappings/Seasons.py +++ b/backend-container/src/apimappings/Seasons.py @@ -1,115 +1,112 @@ -import sys -sys.path.append('..') -import requests, pandas, json -from flask import Blueprint, jsonify, Flask -import os, time -from pymongo import MongoClient -from datetime import datetime -from uuid import UUID -from src.models.seasons import(SeasonInfo) -from mongoengine import connect, DoesNotExist, DecimalField, EmbeddedDocumentField, Document, StringField, UUIDField, IntField, BooleanField, DateTimeField, EmbeddedDocument, EmbeddedDocumentListField -from bson import ObjectId -import logging -bp = Blueprint('Seasons', __name__) -@bp.route('/Seasons', methods=['GET']) -def fetch_and_save_seasons(): - logging.basicConfig(filename='fetch_and_save_seasons.log', level=logging.INFO) - logger = logging.getLogger('fetch_and_save_seasons') - logger.info("fetch_and_save_seasons called") - print("fetch_and_save_seasons called") - url = os.getenv('SEASONS_API') - print(datetime.now(), "Requesting URL:", url) - response = requests.get(url) - print("Response status code:", response.status_code) - if response.status_code != 200: - return f"Call Hierarchy Successfully{response.status_code}" - data = response.json() - season_indo_dict = extract_season_info(data) - mapped_seasons = map_seaoson_info(season_indo_dict) - save_to_database(mapped_seasons) - print("Seasons data fetched and saved successfully.") - return "Seasons data fetched and saved successfully." - -def extract_season_info(data): - # List to hold the SeasonInfo objects - seasons_info_dict = {} - # Extract each season's data and map it to the SeasonInfo model - for season in data['seasons']: # Iterating through list of seasons - season_info = { - 'id': season['id'], - 'year': season['year'], - 'startdate': season['start_date'], - 'enddate': season['end_date'], - 'status': season['status'], - 'type': season['type']['code'] - } - seasons_info_dict[season_info['id']] = season_info - print(f"Number of Seasons extracted: {len(seasons_info_dict)}") - return seasons_info_dict - -def map_seaoson_info(season_info_dict): - mapped_seasons = {} - for season_id, season_data in season_info_dict.items(): - # Debug: Print the type and content of season_details - print("Type of season details:", type(season_data)) - print("Content of season details:", season_data) - season_embedded = SeasonInfo( - id=season_data['id'], - year=season_data['year'], - startdate=season_data['startdate'], - enddate=season_data['enddate'], - status=season_data['status'], - type=season_data['type'] - ) - mapped_seasons[season_id] = season_embedded - - # Debug: Print the number of mapped seasons and their content - print(f"Number of mapped seasons: {len(mapped_seasons)}") - print("Mapped_:", mapped_seasons) - return mapped_seasons - -def save_to_database(mapped_seasons): - print("save_to_database called") - - def update_collection(model_cls, mapped_data, collection_name): - updated_count = 0 - new_count = 0 - - for mapped_entry_id, mapped_entry in mapped_data.items(): - if isinstance(mapped_entry, dict): - new_entry_data = mapped_entry - else: - new_entry_data = mapped_entry.to_mongo().to_dict() - new_entry = model_cls(id=mapped_entry_id, **new_entry_data) - existing_entry = model_cls.objects(_id=mapped_entry_id).first() - print(f"Checking {collection_name} with ID: {mapped_entry_id}") # Print the ID being checked - if existing_entry: - updated_fields = [] - for field_name in existing_entry._fields.keys(): - existing_value = getattr(existing_entry, field_name) - mapped_value = getattr(mapped_entry, field_name) - if existing_value != mapped_value: - setattr(existing_entry, field_name, mapped_value) - updated_fields.append(field_name) - if updated_fields: - existing_entry.save() - updated_count += 1 - print(f"Updated {collection_name} {mapped_entry_id}: Updated fields: {', '.join(updated_fields)}") - else: - print(f"No updates needed for {collection_name} {mapped_entry_id}") - else: - if isinstance(mapped_entry, dict): - new_entry = model_cls(**mapped_entry) - else: - mongo_representation = mapped_entry.to_mongo() - if '_id' in mongo_representation and model_cls == SeasonInfo: - mongo_representation['id'] = mongo_representation.pop('_id') - new_entry = model_cls(**new_entry_data) - new_entry.save() - print(f"Added new {collection_name} with id {new_entry.id}") # Print after saving - new_count += 1 - - print(f"Updated {updated_count} {collection_name}s and added {new_count} new {collection_name}s.") - - # Call the update function for the season collection - update_collection(SeasonInfo, mapped_seasons, "SeasonInfo") +import sys +from security import safe_requests + +sys.path.append('..') +from flask import Blueprint +import os +from datetime import datetime +from src.models.seasons import(SeasonInfo) +import logging +bp = Blueprint('Seasons', __name__) +@bp.route('/Seasons', methods=['GET']) +def fetch_and_save_seasons(): + logging.basicConfig(filename='fetch_and_save_seasons.log', level=logging.INFO) + logger = logging.getLogger('fetch_and_save_seasons') + logger.info("fetch_and_save_seasons called") + print("fetch_and_save_seasons called") + url = os.getenv('SEASONS_API') + print(datetime.now(), "Requesting URL:", url) + response = safe_requests.get(url) + print("Response status code:", response.status_code) + if response.status_code != 200: + return f"Call Hierarchy Successfully{response.status_code}" + data = response.json() + season_indo_dict = extract_season_info(data) + mapped_seasons = map_seaoson_info(season_indo_dict) + save_to_database(mapped_seasons) + print("Seasons data fetched and saved successfully.") + return "Seasons data fetched and saved successfully." + +def extract_season_info(data): + # List to hold the SeasonInfo objects + seasons_info_dict = {} + # Extract each season's data and map it to the SeasonInfo model + for season in data['seasons']: # Iterating through list of seasons + season_info = { + 'id': season['id'], + 'year': season['year'], + 'startdate': season['start_date'], + 'enddate': season['end_date'], + 'status': season['status'], + 'type': season['type']['code'] + } + seasons_info_dict[season_info['id']] = season_info + print(f"Number of Seasons extracted: {len(seasons_info_dict)}") + return seasons_info_dict + +def map_seaoson_info(season_info_dict): + mapped_seasons = {} + for season_id, season_data in season_info_dict.items(): + # Debug: Print the type and content of season_details + print("Type of season details:", type(season_data)) + print("Content of season details:", season_data) + season_embedded = SeasonInfo( + id=season_data['id'], + year=season_data['year'], + startdate=season_data['startdate'], + enddate=season_data['enddate'], + status=season_data['status'], + type=season_data['type'] + ) + mapped_seasons[season_id] = season_embedded + + # Debug: Print the number of mapped seasons and their content + print(f"Number of mapped seasons: {len(mapped_seasons)}") + print("Mapped_:", mapped_seasons) + return mapped_seasons + +def save_to_database(mapped_seasons): + print("save_to_database called") + + def update_collection(model_cls, mapped_data, collection_name): + updated_count = 0 + new_count = 0 + + for mapped_entry_id, mapped_entry in mapped_data.items(): + if isinstance(mapped_entry, dict): + new_entry_data = mapped_entry + else: + new_entry_data = mapped_entry.to_mongo().to_dict() + new_entry = model_cls(id=mapped_entry_id, **new_entry_data) + existing_entry = model_cls.objects(_id=mapped_entry_id).first() + print(f"Checking {collection_name} with ID: {mapped_entry_id}") # Print the ID being checked + if existing_entry: + updated_fields = [] + for field_name in existing_entry._fields.keys(): + existing_value = getattr(existing_entry, field_name) + mapped_value = getattr(mapped_entry, field_name) + if existing_value != mapped_value: + setattr(existing_entry, field_name, mapped_value) + updated_fields.append(field_name) + if updated_fields: + existing_entry.save() + updated_count += 1 + print(f"Updated {collection_name} {mapped_entry_id}: Updated fields: {', '.join(updated_fields)}") + else: + print(f"No updates needed for {collection_name} {mapped_entry_id}") + else: + if isinstance(mapped_entry, dict): + new_entry = model_cls(**mapped_entry) + else: + mongo_representation = mapped_entry.to_mongo() + if '_id' in mongo_representation and model_cls == SeasonInfo: + mongo_representation['id'] = mongo_representation.pop('_id') + new_entry = model_cls(**new_entry_data) + new_entry.save() + print(f"Added new {collection_name} with id {new_entry.id}") # Print after saving + new_count += 1 + + print(f"Updated {updated_count} {collection_name}s and added {new_count} new {collection_name}s.") + + # Call the update function for the season collection + update_collection(SeasonInfo, mapped_seasons, "SeasonInfo") diff --git a/backend-container/src/apimappings/TeamProfile.py b/backend-container/src/apimappings/TeamProfile.py index 9f3bbf3a..391f3ef6 100644 --- a/backend-container/src/apimappings/TeamProfile.py +++ b/backend-container/src/apimappings/TeamProfile.py @@ -1,526 +1,523 @@ -import sys -sys.path.append('..') -sys.path.append("os.getenv('LPATH')/src/") -import requests, pandas, json -from flask import Blueprint, jsonify, Flask -import os, time -if not hasattr(os, 'add_dll_directory'): - def add_dll_directory(path): - pass -from pymongo import MongoClient -from datetime import datetime -from uuid import UUID -from src.models.franchise_info import(FranchiseInfo) -from src.models.venue_info import(venue1, location, VenueInfo) -from src.models.leaguehierarchy import(teams, division, conference, league,typeleague, LeagueHierarchy) -from src.models.team_info import(coach, rgb_color, team, team_color, TeamInfo) -from src.models.player_DCI_info import(player, prospect, primary, position, practice, injury, PlayerDCIinfo) -from mongoengine import DoesNotExist, DecimalField, EmbeddedDocumentField, Document, StringField, UUIDField, IntField, BooleanField, DateTimeField, EmbeddedDocument, EmbeddedDocumentListField -from bson import ObjectId -import logging -bp = Blueprint('TeamProfile', __name__) -@bp.route('/TeamProfile', methods=['GET']) -def fetch_and_save_team_profile(): - logging.basicConfig(filename='fetch_and_save.log', level=logging.INFO) - logger = logging.getLogger('fetch_and_save_team_profile') - logger.info("fetch_and_save_team_profile called") - API_KEY = os.getenv('APIKEY') - URL = "http://api.sportradar.us/nfl/official/trial/v7/en/teams/{TeamID}/profile.json?api_key={API_KEY}" - team_ids = [ - "ce92bd47-93d5-4fe9-ada4-0fc681e6caa0" , - "1f6dcffb-9823-43cd-9ff4-e7a8466749b5" , - "6680d28d-d4d2-49f6-aace-5292d3ec02c2" , - "7d4fcc64-9cb5-4d1b-8e75-8a906d1e1576" , - "768c92aa-75ff-4a43-bcc0-f2798c2e1724" , - "4809ecb0-abd3-451d-9c4a-92a90b83ca06" , - "5fee86ae-74ab-4bdd-8416-42a9dd9964f3" , - "97354895-8c77-4fd4-a860-32e62ea7382a" , - "82cf9565-6eb9-4f01-bdbd-5aa0d472fcd9" , - "f7ddd7fa-0bae-4f90-bc8e-669e4d6cf2de" , - "82d2d380-3834-4938-835f-aec541e5ece7" , - "d26a1ca5-722d-4274-8f97-c92e49c96315" , - "ad4ae08f-d808-42d5-a1e6-e9bc4e34d123" , - "d5a2eb42-8065-4174-ab79-0a6fa820e35e" , - "ebd87119-b331-4469-9ea6-d51fe3ce2f1c" , - "cb2f9f1f-ac67-424e-9e72-1475cb0ed398" , - "4254d319-1bc7-4f81-b4ab-b5e6f3402b69" , - "e6aa13a4-0055-48a9-bc41-be28dc106929" , - "f14bf5cc-9a82-4a38-bc15-d39f75ed5314" , - "0d855753-ea21-4953-89f9-0e20aff9eb73" , - "f0e724b0-4cbf-495a-be47-013907608da9" , - "de760528-1dc0-416a-a978-b510d20692ff" , - "2eff2a03-54d4-46ba-890e-2bc3925548f3" , - "3d08af9e-c767-4f88-a7dc-b920c6d2b4a8" , - "22052ff7-c065-42ee-bc8f-c4691c50e624" , - "e627eec7-bbae-4fa4-8e73-8e1d6bc5c060" , - "386bdbf9-9eea-4869-bb9a-274b0bc66e80" , - "04aa1c9d-66da-489d-b16a-1dee3f2eec4d" , - "7b112545-38e6-483c-a55c-96cf6ee49cb8" , - "c5a59daa-53a7-4de0-851f-fb12be893e9e" , - "a20471b4-a8d9-40c7-95ad-90cc30e46932" , - "33405046-04ee-4058-a950-d606f8c30852" - ] - total_mapped_franchises = 0 - total_mapped_leaguehierarchy = 0 - total_mapped_players = 0 - total_mapped_teams = 0 - total_mapped_venues = 0 - - try: - for TeamID in team_ids: - logger.info(f"{datetime.now()} Requesting URL: {URL.format(TeamID=TeamID)}") - response = requests.get(URL.format(TeamID=TeamID)) - logger.info(f"Response status code: {response.status_code}") - - if response.status_code != 200: - logger.error(f"API request failed for TeamID {TeamID}. Status code: {response.status_code}") - continue - - data = response.json() - - # Extract and map franchise info - franchise_info = extract_franchise_info(data) - mapped_franchises = map_franchise_info(franchise_info) - total_mapped_franchises += 1 - - # Extract and map league hierarchy - league_hierarchy_dict = extract_league_hierarchy(data) - mapped_leaguehierarchy = map_leaguehierarchy(league_hierarchy_dict) - total_mapped_leaguehierarchy += len(mapped_leaguehierarchy['conferences']) - - # Extract and map player info - player_info_dict = extract_player_info(data) - mapped_players = map_player_info(player_info_dict) - total_mapped_players += len(mapped_players) - - # Extract and map team info - team_info_dict = extract_team_info(data) - mapped_teams = map_team_info(team_info_dict) - total_mapped_teams += 1 - - # Extract and map venue info - venue_info_dict = extract_venue_info(data) - mapped_venues = map_venue_info(venue_info_dict) - total_mapped_venues += 1 - - # Save mapped data to the database - save_to_database(mapped_leaguehierarchy, mapped_franchises, mapped_players, mapped_teams, mapped_venues) - logger.info(f"Team Profile Data for TeamID {TeamID} fetched and saved successfully") - - # Add a delay between requests to avoid rate limiting - time.sleep(2) - - logger.info(f"Total mapped franchises: {total_mapped_franchises}") - logger.info(f"Total mapped league hierarchy items: {total_mapped_leaguehierarchy}") - logger.info(f"Total mapped players: {total_mapped_players}") - logger.info(f"Total mapped teams: {total_mapped_teams}") - logger.info(f"Total mapped venues: {total_mapped_venues}") - return "Team Profile Data Fetched and Saved Successfully" - except Exception as e: - logger.exception("An error occurred:") - return f"Error: {str(e)}", 500 -def extract_franchise_info(data): - return { - 'id': data['id'], - 'falias': data['franchise']['alias'], - 'fid': data['franchise']['id'], - 'fname': data['franchise']['name'], - } -def map_franchise_info(franchise_info): - franchise_id = franchise_info['fid'] - mapped_franchise = FranchiseInfo( - teamid=franchise_info['id'], - alias=franchise_info['falias'], - id=franchise_id, - name=franchise_info['fname'] - ) - return {franchise_id: mapped_franchise} -def extract_league_hierarchy(data): - # Extract conference and division data from raw data - conference_data = data['conference'] - division_data = data['division'] - - # Extract specific data points - extracted_conference = { - 'calias': conference_data['alias'], - 'cid': conference_data['id'], - 'cname': conference_data['name'], - 'divisions': [ - { - 'dalias': division_data['alias'], - 'did': division_data['id'], - 'dname': division_data['name'] - } - ] - } - - return {'conferences': [extracted_conference]} -def map_leaguehierarchy(league_hierarchy_dict): - # Manually set the static ID for the mapped_leaguehierarchy - static_id = "3c6d318a-6164-4290-9bbc-bf9bb21cc4b8" - # Create a league embedded document instance - mapped_league = { - 'id': static_id, - 'alias': "NFL", - 'name': "National Football League", - 'conferences': [] - } - mapped_leaguehierarchy = { - 'id': static_id, - 'league_id': static_id, # Set league_id to the same value as id - 'conferences': {} - } - for conference_dict in league_hierarchy_dict.get('conferences', []): - cid = conference_dict.get('cid') - calias = conference_dict.get('calias') - cname = conference_dict.get('cname') - for division_dict in conference_dict.get('divisions', []): - did = division_dict.get('did') - dalias = division_dict.get('dalias') - dname = division_dict.get('dname') - division_key = (cid, did) - mapped_division = { - 'did': did, - 'dalias': dalias, - 'dname': dname, - } - if division_key not in mapped_leaguehierarchy['conferences']: - mapped_leaguehierarchy['conferences'][division_key] = { - 'cid': cid, - 'calias': calias, - 'cname': cname, - 'divisions': [mapped_division], - } - else: - mapped_leaguehierarchy['conferences'][division_key]['divisions'].append(mapped_division) - # Assign the league embedded document instance to the league attribute - mapped_leaguehierarchy['league'] = mapped_league - logging.info("Mapped League Hierarchy:", mapped_leaguehierarchy) - return mapped_leaguehierarchy -def extract_player_info(data): - players_info = data.get("players", []) - extracted_players = {} - for player_info in players_info: - logging.info(f"Raw player data: {player_info}") - player_id = player_info.get('id') - logging.info(f"Extracted player ID: {player_id}") - extracted_player = { - "abbrname": player_info.get("abbr_name"), - "birthdate": player_info.get("birth_date"), - "birthplace": player_info.get("birth_place"), - "college": player_info.get("college"), - "collegeconf": player_info.get("college_conf"), - "draftinfonumber": player_info.get("draft", {}).get("number"), - "draftinfo_round": player_info.get("draft", {}).get("round"), - "draftinfo_year": player_info.get("draft", {}).get("year"), - "draftteamalias": player_info.get("draft", {}).get("team", {}).get("alias"), - "draftteamid": player_info.get("draft", {}).get("team", {}).get("id"), - "draftteammarket": player_info.get("draft", {}).get("team", {}).get("market"), - "draftteamname": player_info.get("draft", {}).get("team", {}).get("name"), - "draftteamsrid": player_info.get("draft", {}).get("team", {}).get("sr_id"), - "experience": player_info.get("experience"), - "firstname": player_info.get("first_name"), - "fullname": f"{player_info.get('first_name')} {player_info.get('last_name')}", # Constructed fullname - "height": player_info.get("height"), - "highschool": player_info.get("high_school"), - "id": player_info.get("id"), - "jerseynumber": player_info.get("jersey"), - "lastname": player_info.get("last_name"), - "namesuffix": player_info.get("name_suffix"), - "position": player_info.get("position"), - "preferredname": player_info.get("name"), - "srid": player_info.get("sr_id"), - "status": player_info.get("status"), - "weight": player_info.get("weight"), - } - extracted_players[player_id] = extracted_player - logging.info(f"Number of players extracted: {len(extracted_players)}") - return extracted_players -def map_player_info(extracted_players): - mapped_players = {} - for player_id, player_details in extracted_players.items(): - try: - playerembedded = player( - abbrname=player_details.get('abbrname', ''), - age=player_details.get('age', None), - birthdate=player_details.get('birthdate', ''), - birthplace=player_details.get('birthplace', ''), - id=player_details.get('id', ''), - college=player_details.get('college', ''), - collegeconf=player_details.get('collegeconf', ''), - depth=player_details.get('depth', None), - experience=player_details.get('experience', 0), - firstname=player_details.get('firstname', ''), - fullname=player_details.get('fullname', ''), - height=player_details.get('height', 0), - highschool=player_details.get('highschool', ''), - srid=player_details.get('srid', ''), - ingamestatus=player_details.get('ingamestatus', None), - jersey=player_details.get('jersey', ''), - lastmodified=player_details.get('lastmodified', None), - lastname=player_details.get('lastname', ''), - status=player_details.get('status', ''), - namesuffix=player_details.get('namesuffix', None), - position=player_details.get('position', ''), - preferredname=player_details.get('preferredname', ''), - role=player_details.get('role', None), - sourceid=player_details.get('sourceid', None), - weight=player_details.get('weight', 0), - draftyear=player_details.get('draftyear', None), - draftround=player_details.get('draftround', None), - draftnumber=player_details.get('draftnumber', None), - draftteamid=player_details.get('draftteamid', None), - draftteamname=player_details.get('draftteamname', None), - draftteamalias=player_details.get('draftteamalias', None), - draftteamsrid=player_details.get('draftteamsrid', None), - draftteammarket=player_details.get('draftteammarket', None) - ) - player_info_instance = PlayerDCIinfo( - playerinfo=playerembedded, - prospectinfo=None, - primaryposition=None, - positioninfo=None, - practiceinfo=None, - injuryinfo=None - ) - mapped_players[player_id] = player_info_instance - except Exception as e: - logging.info(f"Error mapping player ID {player_id}. Error: {e}") - - logging.info(f"Number of mapped players: {len(mapped_players)}") - logging.info("mapped_players:", mapped_players) - return mapped_players -def extract_team_info(data): - team_colors = data.get('team_colors', []) - team_info = { - 'alias': data['alias'], - 'coaches': [], - 'id': data['id'], - 'market': data['market'], - 'name': data['name'], - 'sr_id': data['sr_id'], - 'team_colors': [], # Placeholder for colors information - 'conference_id': data['conference']['id'], # Extract conference id - 'division_id': data['division']['id'] # Extract division id - } - # Extract coach information - for coach in data['coaches']: - coach_info = { - 'coach_first_name': coach['first_name'], - 'coach_full_name': coach['full_name'], - 'coach_id': coach['id'], - 'coach_last_name': coach['last_name'], - 'coach_name_suffix': coach.get('name_suffix', ''), - 'coach_position': coach['position'] - } - team_info['coaches'].append(coach_info) - # Extract team color information if available - for color in team_colors: - color_info = { - 'color_type': color.get('type', ''), - 'color_hex': color.get('hex_color', ''), - 'color_alpha': color.get('alpha', ''), - 'color_rgb_blue': color.get('rgb_color', {}).get('blue', ''), - 'color_rgb_green': color.get('rgb_color', {}).get('green', ''), - 'color_rgb_red': color.get('rgb_color', {}).get('red', ''), - } - team_info['team_colors'].append(color_info) - return team_info -def map_team_info(team_info): - team_id = team_info['id'] - # Mapping coaches - mapped_coaches = [coach( - first_name=coach_data['coach_first_name'], - full_name=coach_data['coach_full_name'], - id=coach_data['coach_id'], - last_name=coach_data['coach_last_name'], - name_suffix=coach_data['coach_name_suffix'], - position=coach_data['coach_position'] - ) for coach_data in team_info['coaches']] - # Mapping RGB colors - mapped_rgb_colors = [rgb_color( - blue=color_data['color_rgb_blue'], - green=color_data['color_rgb_green'], - red=color_data['color_rgb_red'] - ) for color_data in team_info['team_colors']] - # Mapping team colors - mapped_team_colors = [team_color( - alpha=color_data['color_alpha'], - hex_color=color_data['color_hex'], - type=color_data['color_type'] - ) for color_data in team_info['team_colors']] - # Mapping team - mapped_team = team( - alias=team_info['alias'], - id=team_id, - market=team_info['market'], - name=team_info['name'], - sr_id=team_info['sr_id'] - ) - return { - team_id: { - "team": mapped_team, - "coachs": mapped_coaches, - "rgb_colors": mapped_rgb_colors, - "team_colors": mapped_team_colors, - "conference_id": team_info.get('conference_id', None), - "division_id": team_info.get('division_id', None), - } - } -def extract_venue_info(data): - venue_data = data.get('venue', {}) - # Debug: Print the type and content of venue_data to check its structure - logging.info("Type of venue data:", type(venue_data)) - logging.info("Content of venue data:", venue_data) - venue_info = { - 'id': venue_data.get('id'), - 'name': venue_data.get('name'), - 'city': venue_data.get('city'), - 'state': venue_data.get('state'), - 'country': venue_data.get('country'), - 'zip': venue_data.get('zip'), - 'address': venue_data.get('address'), - 'capacity': venue_data.get('capacity'), - 'surface': venue_data.get('surface'), - 'roof_type': venue_data.get('roof_type'), - 'sr_id': venue_data.get('sr_id'), - 'location': venue_data.get('location', {}), - } - # Debug: Print the type and content of venue_info - logging.info("Type of venue info:", type(venue_info)) - logging.info("Content of venue info:", venue_info) - return venue_info -def map_venue_info(venue_info_dict): - # No need to loop; just extract the details from venue_info_dict directly - venue_details = venue_info_dict - # Debug: Print the type and content of venue_details - logging.info("Type of venue details:", type(venue_details)) - logging.info("Content of venue details:", venue_details) - venue_embedded = venue1( - id=venue_details['id'], - address=venue_details['address'], - capacity=venue_details['capacity'], - city=venue_details['city'], - country=venue_details['country'], - name=venue_details['name'], - sr_id=venue_details['sr_id'], - roof_type=venue_details['roof_type'], - state=venue_details['state'], - surface=venue_details['surface'], - zip=venue_details['zip'] - ) - location_embedded = location( - lat=venue_details['location']['lat'], - lng=venue_details['location']['lng'] - ) - venue_info_instance = VenueInfo( - venue1=venue_embedded, - location=location_embedded - ) - mapped_venues = {venue_details['id']: venue_info_instance} - # Debug: Print the number of mapped venues and their content - logging.info(f"Number of mapped venues: {len(mapped_venues)}") - logging.info("Mapped_Venues:", mapped_venues) - return mapped_venues -def map_league_hierarchy(league_hierarchy_dict): - # Create an empty LeagueHierarchy instance - mapped_league_hierarchy = LeagueHierarchy() - # Create a list to store mapped conferences - mapped_conferences_list = [] - for conference_dict in league_hierarchy_dict.get('conferences', []): - # Extract conference details - cid = conference_dict.get('cid') - calias = conference_dict.get('calias') - cname = conference_dict.get('cname') - # Create a list to store mapped divisions for this conference - mapped_divisions_list = [] - for division_dict in conference_dict.get('divisions', []): - # Extract division details - did = division_dict.get('did') - dalias = division_dict.get('dalias') - dname = division_dict.get('dname') - # Create division instance - mapped_division = division(did=did, dalias=dalias, dname=dname) - # Append to the divisions list - mapped_divisions_list.append(mapped_division) - # Create conference instance with the divisions list - mapped_conference = conference(cid=cid, calias=calias, cname=cname, divisions=mapped_divisions_list) - # Append to the conferences list - mapped_conferences_list.append(mapped_conference) - # Assign the mapped conferences list to the LeagueHierarchy instance - mapped_league_hierarchy.conferences = mapped_conferences_list - return mapped_league_hierarchy -def save_to_database(mapped_leaguehierarchy, mapped_franchises, mapped_players, mapped_teams, mapped_venues): - logging.info("save_to_database called") - - def update_collection(model_cls, mapped_data, collection_name): - updated_count = 0 - new_count = 0 - if isinstance(mapped_data, dict): - mapped_data_iterable = mapped_data.items() - else: - mapped_data_iterable = ((getattr(entry, 'id', None), entry) for entry in mapped_data) - for mapped_entry_id, mapped_entry in mapped_data_iterable: - if isinstance(mapped_entry, dict) and 'team' in mapped_entry: - mapped_entry = mapped_entry['team'] - if isinstance(mapped_entry, tuple): - _, mapped_entry = mapped_entry - existing_entry = model_cls.objects(id=mapped_entry_id).first() - logging.info(f"Checking {collection_name} with ID: {mapped_entry_id}") # Print the ID being checked - if existing_entry: - updated_fields = [] - for field_name in existing_entry._fields.keys(): - existing_value = getattr(existing_entry, field_name) - mapped_value = getattr(mapped_entry, field_name) - if existing_value != mapped_value: - setattr(existing_entry, field_name, mapped_value) - updated_fields.append(field_name) - if updated_fields: - existing_entry.save() - updated_count += 1 - logging.info(f"Updated {collection_name} {mapped_entry_id}: Updated fields: {', '.join(updated_fields)}") - else: - logging.info(f"No updates needed for {collection_name} {mapped_entry_id}") - else: - logging.info(f"Type of mapped_entry: {type(mapped_entry)}, Content: {mapped_entry}") - if model_cls == PlayerDCIinfo: - logging.info(f"Player ID from mapped data: {mapped_entry.id}") - mapped_entry = mapped_entry.to_mongo().to_dict() - if isinstance(mapped_entry, dict): - new_entry = model_cls(**mapped_entry) - mongo_representation = new_entry.to_mongo() - else: - mongo_representation = mapped_entry.to_mongo() - if '_id' in mongo_representation and model_cls == FranchiseInfo: - mongo_representation['id'] = mongo_representation.pop('_id') - new_entry = model_cls(**mongo_representation) - new_entry.save() - logging.info(f"Added new {collection_name} with id {new_entry.id}") # Print after saving - new_count += 1 - logging.info(f"Updated {updated_count} {collection_name}s and added {new_count} new {collection_name}s.") - - update_collection(FranchiseInfo, mapped_franchises.values(), "franchise") - update_collection(PlayerDCIinfo, mapped_players.values(), "player") - update_collection(TeamInfo, mapped_teams.items(), "teams") - update_collection(VenueInfo, mapped_venues.items(), "venue") - - # Handling the LeagueHierarchy separately, as there's only one document. - existing_league_hierarchy = LeagueHierarchy.objects.first() - if existing_league_hierarchy: - # Update the existing document - updated_fields = [] - fields_to_skip = ['id', 'league_id'] - for field_name in existing_league_hierarchy._fields.keys(): - if field_name not in fields_to_skip: - existing_value = getattr(existing_league_hierarchy, field_name) - mapped_value = mapped_leaguehierarchy.get(field_name) - if existing_value != mapped_value: - setattr(existing_league_hierarchy, field_name, mapped_value) - updated_fields.append(field_name) - # Save the updated LeagueHierarchy document - existing_league_hierarchy.save() - if updated_fields: - logging.info(f"Updated the LeagueHierarchy: Updated fields: {', '.join(updated_fields)}") - else: - logging.info("No updates needed for LeagueHierarchy") - +import sys +from security import safe_requests + +sys.path.append('..') +sys.path.append("os.getenv('LPATH')/src/") +from flask import Blueprint +import os, time +if not hasattr(os, 'add_dll_directory'): + def add_dll_directory(path): + pass +from datetime import datetime +from src.models.franchise_info import(FranchiseInfo) +from src.models.venue_info import(venue1, location, VenueInfo) +from src.models.leaguehierarchy import(division, conference, LeagueHierarchy) +from src.models.team_info import(coach, rgb_color, team, team_color, TeamInfo) +from src.models.player_DCI_info import(player, PlayerDCIinfo) +import logging +bp = Blueprint('TeamProfile', __name__) +@bp.route('/TeamProfile', methods=['GET']) +def fetch_and_save_team_profile(): + logging.basicConfig(filename='fetch_and_save.log', level=logging.INFO) + logger = logging.getLogger('fetch_and_save_team_profile') + logger.info("fetch_and_save_team_profile called") + API_KEY = os.getenv('APIKEY') + URL = "http://api.sportradar.us/nfl/official/trial/v7/en/teams/{TeamID}/profile.json?api_key={API_KEY}" + team_ids = [ + "ce92bd47-93d5-4fe9-ada4-0fc681e6caa0" , + "1f6dcffb-9823-43cd-9ff4-e7a8466749b5" , + "6680d28d-d4d2-49f6-aace-5292d3ec02c2" , + "7d4fcc64-9cb5-4d1b-8e75-8a906d1e1576" , + "768c92aa-75ff-4a43-bcc0-f2798c2e1724" , + "4809ecb0-abd3-451d-9c4a-92a90b83ca06" , + "5fee86ae-74ab-4bdd-8416-42a9dd9964f3" , + "97354895-8c77-4fd4-a860-32e62ea7382a" , + "82cf9565-6eb9-4f01-bdbd-5aa0d472fcd9" , + "f7ddd7fa-0bae-4f90-bc8e-669e4d6cf2de" , + "82d2d380-3834-4938-835f-aec541e5ece7" , + "d26a1ca5-722d-4274-8f97-c92e49c96315" , + "ad4ae08f-d808-42d5-a1e6-e9bc4e34d123" , + "d5a2eb42-8065-4174-ab79-0a6fa820e35e" , + "ebd87119-b331-4469-9ea6-d51fe3ce2f1c" , + "cb2f9f1f-ac67-424e-9e72-1475cb0ed398" , + "4254d319-1bc7-4f81-b4ab-b5e6f3402b69" , + "e6aa13a4-0055-48a9-bc41-be28dc106929" , + "f14bf5cc-9a82-4a38-bc15-d39f75ed5314" , + "0d855753-ea21-4953-89f9-0e20aff9eb73" , + "f0e724b0-4cbf-495a-be47-013907608da9" , + "de760528-1dc0-416a-a978-b510d20692ff" , + "2eff2a03-54d4-46ba-890e-2bc3925548f3" , + "3d08af9e-c767-4f88-a7dc-b920c6d2b4a8" , + "22052ff7-c065-42ee-bc8f-c4691c50e624" , + "e627eec7-bbae-4fa4-8e73-8e1d6bc5c060" , + "386bdbf9-9eea-4869-bb9a-274b0bc66e80" , + "04aa1c9d-66da-489d-b16a-1dee3f2eec4d" , + "7b112545-38e6-483c-a55c-96cf6ee49cb8" , + "c5a59daa-53a7-4de0-851f-fb12be893e9e" , + "a20471b4-a8d9-40c7-95ad-90cc30e46932" , + "33405046-04ee-4058-a950-d606f8c30852" + ] + total_mapped_franchises = 0 + total_mapped_leaguehierarchy = 0 + total_mapped_players = 0 + total_mapped_teams = 0 + total_mapped_venues = 0 + + try: + for TeamID in team_ids: + logger.info(f"{datetime.now()} Requesting URL: {URL.format(TeamID=TeamID)}") + response = safe_requests.get(URL.format(TeamID=TeamID)) + logger.info(f"Response status code: {response.status_code}") + + if response.status_code != 200: + logger.error(f"API request failed for TeamID {TeamID}. Status code: {response.status_code}") + continue + + data = response.json() + + # Extract and map franchise info + franchise_info = extract_franchise_info(data) + mapped_franchises = map_franchise_info(franchise_info) + total_mapped_franchises += 1 + + # Extract and map league hierarchy + league_hierarchy_dict = extract_league_hierarchy(data) + mapped_leaguehierarchy = map_leaguehierarchy(league_hierarchy_dict) + total_mapped_leaguehierarchy += len(mapped_leaguehierarchy['conferences']) + + # Extract and map player info + player_info_dict = extract_player_info(data) + mapped_players = map_player_info(player_info_dict) + total_mapped_players += len(mapped_players) + + # Extract and map team info + team_info_dict = extract_team_info(data) + mapped_teams = map_team_info(team_info_dict) + total_mapped_teams += 1 + + # Extract and map venue info + venue_info_dict = extract_venue_info(data) + mapped_venues = map_venue_info(venue_info_dict) + total_mapped_venues += 1 + + # Save mapped data to the database + save_to_database(mapped_leaguehierarchy, mapped_franchises, mapped_players, mapped_teams, mapped_venues) + logger.info(f"Team Profile Data for TeamID {TeamID} fetched and saved successfully") + + # Add a delay between requests to avoid rate limiting + time.sleep(2) + + logger.info(f"Total mapped franchises: {total_mapped_franchises}") + logger.info(f"Total mapped league hierarchy items: {total_mapped_leaguehierarchy}") + logger.info(f"Total mapped players: {total_mapped_players}") + logger.info(f"Total mapped teams: {total_mapped_teams}") + logger.info(f"Total mapped venues: {total_mapped_venues}") + return "Team Profile Data Fetched and Saved Successfully" + except Exception as e: + logger.exception("An error occurred:") + return f"Error: {str(e)}", 500 +def extract_franchise_info(data): + return { + 'id': data['id'], + 'falias': data['franchise']['alias'], + 'fid': data['franchise']['id'], + 'fname': data['franchise']['name'], + } +def map_franchise_info(franchise_info): + franchise_id = franchise_info['fid'] + mapped_franchise = FranchiseInfo( + teamid=franchise_info['id'], + alias=franchise_info['falias'], + id=franchise_id, + name=franchise_info['fname'] + ) + return {franchise_id: mapped_franchise} +def extract_league_hierarchy(data): + # Extract conference and division data from raw data + conference_data = data['conference'] + division_data = data['division'] + + # Extract specific data points + extracted_conference = { + 'calias': conference_data['alias'], + 'cid': conference_data['id'], + 'cname': conference_data['name'], + 'divisions': [ + { + 'dalias': division_data['alias'], + 'did': division_data['id'], + 'dname': division_data['name'] + } + ] + } + + return {'conferences': [extracted_conference]} +def map_leaguehierarchy(league_hierarchy_dict): + # Manually set the static ID for the mapped_leaguehierarchy + static_id = "3c6d318a-6164-4290-9bbc-bf9bb21cc4b8" + # Create a league embedded document instance + mapped_league = { + 'id': static_id, + 'alias': "NFL", + 'name': "National Football League", + 'conferences': [] + } + mapped_leaguehierarchy = { + 'id': static_id, + 'league_id': static_id, # Set league_id to the same value as id + 'conferences': {} + } + for conference_dict in league_hierarchy_dict.get('conferences', []): + cid = conference_dict.get('cid') + calias = conference_dict.get('calias') + cname = conference_dict.get('cname') + for division_dict in conference_dict.get('divisions', []): + did = division_dict.get('did') + dalias = division_dict.get('dalias') + dname = division_dict.get('dname') + division_key = (cid, did) + mapped_division = { + 'did': did, + 'dalias': dalias, + 'dname': dname, + } + if division_key not in mapped_leaguehierarchy['conferences']: + mapped_leaguehierarchy['conferences'][division_key] = { + 'cid': cid, + 'calias': calias, + 'cname': cname, + 'divisions': [mapped_division], + } + else: + mapped_leaguehierarchy['conferences'][division_key]['divisions'].append(mapped_division) + # Assign the league embedded document instance to the league attribute + mapped_leaguehierarchy['league'] = mapped_league + logging.info("Mapped League Hierarchy:", mapped_leaguehierarchy) + return mapped_leaguehierarchy +def extract_player_info(data): + players_info = data.get("players", []) + extracted_players = {} + for player_info in players_info: + logging.info(f"Raw player data: {player_info}") + player_id = player_info.get('id') + logging.info(f"Extracted player ID: {player_id}") + extracted_player = { + "abbrname": player_info.get("abbr_name"), + "birthdate": player_info.get("birth_date"), + "birthplace": player_info.get("birth_place"), + "college": player_info.get("college"), + "collegeconf": player_info.get("college_conf"), + "draftinfonumber": player_info.get("draft", {}).get("number"), + "draftinfo_round": player_info.get("draft", {}).get("round"), + "draftinfo_year": player_info.get("draft", {}).get("year"), + "draftteamalias": player_info.get("draft", {}).get("team", {}).get("alias"), + "draftteamid": player_info.get("draft", {}).get("team", {}).get("id"), + "draftteammarket": player_info.get("draft", {}).get("team", {}).get("market"), + "draftteamname": player_info.get("draft", {}).get("team", {}).get("name"), + "draftteamsrid": player_info.get("draft", {}).get("team", {}).get("sr_id"), + "experience": player_info.get("experience"), + "firstname": player_info.get("first_name"), + "fullname": f"{player_info.get('first_name')} {player_info.get('last_name')}", # Constructed fullname + "height": player_info.get("height"), + "highschool": player_info.get("high_school"), + "id": player_info.get("id"), + "jerseynumber": player_info.get("jersey"), + "lastname": player_info.get("last_name"), + "namesuffix": player_info.get("name_suffix"), + "position": player_info.get("position"), + "preferredname": player_info.get("name"), + "srid": player_info.get("sr_id"), + "status": player_info.get("status"), + "weight": player_info.get("weight"), + } + extracted_players[player_id] = extracted_player + logging.info(f"Number of players extracted: {len(extracted_players)}") + return extracted_players +def map_player_info(extracted_players): + mapped_players = {} + for player_id, player_details in extracted_players.items(): + try: + playerembedded = player( + abbrname=player_details.get('abbrname', ''), + age=player_details.get('age', None), + birthdate=player_details.get('birthdate', ''), + birthplace=player_details.get('birthplace', ''), + id=player_details.get('id', ''), + college=player_details.get('college', ''), + collegeconf=player_details.get('collegeconf', ''), + depth=player_details.get('depth', None), + experience=player_details.get('experience', 0), + firstname=player_details.get('firstname', ''), + fullname=player_details.get('fullname', ''), + height=player_details.get('height', 0), + highschool=player_details.get('highschool', ''), + srid=player_details.get('srid', ''), + ingamestatus=player_details.get('ingamestatus', None), + jersey=player_details.get('jersey', ''), + lastmodified=player_details.get('lastmodified', None), + lastname=player_details.get('lastname', ''), + status=player_details.get('status', ''), + namesuffix=player_details.get('namesuffix', None), + position=player_details.get('position', ''), + preferredname=player_details.get('preferredname', ''), + role=player_details.get('role', None), + sourceid=player_details.get('sourceid', None), + weight=player_details.get('weight', 0), + draftyear=player_details.get('draftyear', None), + draftround=player_details.get('draftround', None), + draftnumber=player_details.get('draftnumber', None), + draftteamid=player_details.get('draftteamid', None), + draftteamname=player_details.get('draftteamname', None), + draftteamalias=player_details.get('draftteamalias', None), + draftteamsrid=player_details.get('draftteamsrid', None), + draftteammarket=player_details.get('draftteammarket', None) + ) + player_info_instance = PlayerDCIinfo( + playerinfo=playerembedded, + prospectinfo=None, + primaryposition=None, + positioninfo=None, + practiceinfo=None, + injuryinfo=None + ) + mapped_players[player_id] = player_info_instance + except Exception as e: + logging.info(f"Error mapping player ID {player_id}. Error: {e}") + + logging.info(f"Number of mapped players: {len(mapped_players)}") + logging.info("mapped_players:", mapped_players) + return mapped_players +def extract_team_info(data): + team_colors = data.get('team_colors', []) + team_info = { + 'alias': data['alias'], + 'coaches': [], + 'id': data['id'], + 'market': data['market'], + 'name': data['name'], + 'sr_id': data['sr_id'], + 'team_colors': [], # Placeholder for colors information + 'conference_id': data['conference']['id'], # Extract conference id + 'division_id': data['division']['id'] # Extract division id + } + # Extract coach information + for coach in data['coaches']: + coach_info = { + 'coach_first_name': coach['first_name'], + 'coach_full_name': coach['full_name'], + 'coach_id': coach['id'], + 'coach_last_name': coach['last_name'], + 'coach_name_suffix': coach.get('name_suffix', ''), + 'coach_position': coach['position'] + } + team_info['coaches'].append(coach_info) + # Extract team color information if available + for color in team_colors: + color_info = { + 'color_type': color.get('type', ''), + 'color_hex': color.get('hex_color', ''), + 'color_alpha': color.get('alpha', ''), + 'color_rgb_blue': color.get('rgb_color', {}).get('blue', ''), + 'color_rgb_green': color.get('rgb_color', {}).get('green', ''), + 'color_rgb_red': color.get('rgb_color', {}).get('red', ''), + } + team_info['team_colors'].append(color_info) + return team_info +def map_team_info(team_info): + team_id = team_info['id'] + # Mapping coaches + mapped_coaches = [coach( + first_name=coach_data['coach_first_name'], + full_name=coach_data['coach_full_name'], + id=coach_data['coach_id'], + last_name=coach_data['coach_last_name'], + name_suffix=coach_data['coach_name_suffix'], + position=coach_data['coach_position'] + ) for coach_data in team_info['coaches']] + # Mapping RGB colors + mapped_rgb_colors = [rgb_color( + blue=color_data['color_rgb_blue'], + green=color_data['color_rgb_green'], + red=color_data['color_rgb_red'] + ) for color_data in team_info['team_colors']] + # Mapping team colors + mapped_team_colors = [team_color( + alpha=color_data['color_alpha'], + hex_color=color_data['color_hex'], + type=color_data['color_type'] + ) for color_data in team_info['team_colors']] + # Mapping team + mapped_team = team( + alias=team_info['alias'], + id=team_id, + market=team_info['market'], + name=team_info['name'], + sr_id=team_info['sr_id'] + ) + return { + team_id: { + "team": mapped_team, + "coachs": mapped_coaches, + "rgb_colors": mapped_rgb_colors, + "team_colors": mapped_team_colors, + "conference_id": team_info.get('conference_id', None), + "division_id": team_info.get('division_id', None), + } + } +def extract_venue_info(data): + venue_data = data.get('venue', {}) + # Debug: Print the type and content of venue_data to check its structure + logging.info("Type of venue data:", type(venue_data)) + logging.info("Content of venue data:", venue_data) + venue_info = { + 'id': venue_data.get('id'), + 'name': venue_data.get('name'), + 'city': venue_data.get('city'), + 'state': venue_data.get('state'), + 'country': venue_data.get('country'), + 'zip': venue_data.get('zip'), + 'address': venue_data.get('address'), + 'capacity': venue_data.get('capacity'), + 'surface': venue_data.get('surface'), + 'roof_type': venue_data.get('roof_type'), + 'sr_id': venue_data.get('sr_id'), + 'location': venue_data.get('location', {}), + } + # Debug: Print the type and content of venue_info + logging.info("Type of venue info:", type(venue_info)) + logging.info("Content of venue info:", venue_info) + return venue_info +def map_venue_info(venue_info_dict): + # No need to loop; just extract the details from venue_info_dict directly + venue_details = venue_info_dict + # Debug: Print the type and content of venue_details + logging.info("Type of venue details:", type(venue_details)) + logging.info("Content of venue details:", venue_details) + venue_embedded = venue1( + id=venue_details['id'], + address=venue_details['address'], + capacity=venue_details['capacity'], + city=venue_details['city'], + country=venue_details['country'], + name=venue_details['name'], + sr_id=venue_details['sr_id'], + roof_type=venue_details['roof_type'], + state=venue_details['state'], + surface=venue_details['surface'], + zip=venue_details['zip'] + ) + location_embedded = location( + lat=venue_details['location']['lat'], + lng=venue_details['location']['lng'] + ) + venue_info_instance = VenueInfo( + venue1=venue_embedded, + location=location_embedded + ) + mapped_venues = {venue_details['id']: venue_info_instance} + # Debug: Print the number of mapped venues and their content + logging.info(f"Number of mapped venues: {len(mapped_venues)}") + logging.info("Mapped_Venues:", mapped_venues) + return mapped_venues +def map_league_hierarchy(league_hierarchy_dict): + # Create an empty LeagueHierarchy instance + mapped_league_hierarchy = LeagueHierarchy() + # Create a list to store mapped conferences + mapped_conferences_list = [] + for conference_dict in league_hierarchy_dict.get('conferences', []): + # Extract conference details + cid = conference_dict.get('cid') + calias = conference_dict.get('calias') + cname = conference_dict.get('cname') + # Create a list to store mapped divisions for this conference + mapped_divisions_list = [] + for division_dict in conference_dict.get('divisions', []): + # Extract division details + did = division_dict.get('did') + dalias = division_dict.get('dalias') + dname = division_dict.get('dname') + # Create division instance + mapped_division = division(did=did, dalias=dalias, dname=dname) + # Append to the divisions list + mapped_divisions_list.append(mapped_division) + # Create conference instance with the divisions list + mapped_conference = conference(cid=cid, calias=calias, cname=cname, divisions=mapped_divisions_list) + # Append to the conferences list + mapped_conferences_list.append(mapped_conference) + # Assign the mapped conferences list to the LeagueHierarchy instance + mapped_league_hierarchy.conferences = mapped_conferences_list + return mapped_league_hierarchy +def save_to_database(mapped_leaguehierarchy, mapped_franchises, mapped_players, mapped_teams, mapped_venues): + logging.info("save_to_database called") + + def update_collection(model_cls, mapped_data, collection_name): + updated_count = 0 + new_count = 0 + if isinstance(mapped_data, dict): + mapped_data_iterable = mapped_data.items() + else: + mapped_data_iterable = ((getattr(entry, 'id', None), entry) for entry in mapped_data) + for mapped_entry_id, mapped_entry in mapped_data_iterable: + if isinstance(mapped_entry, dict) and 'team' in mapped_entry: + mapped_entry = mapped_entry['team'] + if isinstance(mapped_entry, tuple): + _, mapped_entry = mapped_entry + existing_entry = model_cls.objects(id=mapped_entry_id).first() + logging.info(f"Checking {collection_name} with ID: {mapped_entry_id}") # Print the ID being checked + if existing_entry: + updated_fields = [] + for field_name in existing_entry._fields.keys(): + existing_value = getattr(existing_entry, field_name) + mapped_value = getattr(mapped_entry, field_name) + if existing_value != mapped_value: + setattr(existing_entry, field_name, mapped_value) + updated_fields.append(field_name) + if updated_fields: + existing_entry.save() + updated_count += 1 + logging.info(f"Updated {collection_name} {mapped_entry_id}: Updated fields: {', '.join(updated_fields)}") + else: + logging.info(f"No updates needed for {collection_name} {mapped_entry_id}") + else: + logging.info(f"Type of mapped_entry: {type(mapped_entry)}, Content: {mapped_entry}") + if model_cls == PlayerDCIinfo: + logging.info(f"Player ID from mapped data: {mapped_entry.id}") + mapped_entry = mapped_entry.to_mongo().to_dict() + if isinstance(mapped_entry, dict): + new_entry = model_cls(**mapped_entry) + mongo_representation = new_entry.to_mongo() + else: + mongo_representation = mapped_entry.to_mongo() + if '_id' in mongo_representation and model_cls == FranchiseInfo: + mongo_representation['id'] = mongo_representation.pop('_id') + new_entry = model_cls(**mongo_representation) + new_entry.save() + logging.info(f"Added new {collection_name} with id {new_entry.id}") # Print after saving + new_count += 1 + logging.info(f"Updated {updated_count} {collection_name}s and added {new_count} new {collection_name}s.") + + update_collection(FranchiseInfo, mapped_franchises.values(), "franchise") + update_collection(PlayerDCIinfo, mapped_players.values(), "player") + update_collection(TeamInfo, mapped_teams.items(), "teams") + update_collection(VenueInfo, mapped_venues.items(), "venue") + + # Handling the LeagueHierarchy separately, as there's only one document. + existing_league_hierarchy = LeagueHierarchy.objects.first() + if existing_league_hierarchy: + # Update the existing document + updated_fields = [] + fields_to_skip = ['id', 'league_id'] + for field_name in existing_league_hierarchy._fields.keys(): + if field_name not in fields_to_skip: + existing_value = getattr(existing_league_hierarchy, field_name) + mapped_value = mapped_leaguehierarchy.get(field_name) + if existing_value != mapped_value: + setattr(existing_league_hierarchy, field_name, mapped_value) + updated_fields.append(field_name) + # Save the updated LeagueHierarchy document + existing_league_hierarchy.save() + if updated_fields: + logging.info(f"Updated the LeagueHierarchy: Updated fields: {', '.join(updated_fields)}") + else: + logging.info("No updates needed for LeagueHierarchy") + diff --git a/backend-container/src/apimappings/current_season_schedule.py b/backend-container/src/apimappings/current_season_schedule.py index d2227401..1b6dace0 100644 --- a/backend-container/src/apimappings/current_season_schedule.py +++ b/backend-container/src/apimappings/current_season_schedule.py @@ -1,518 +1,514 @@ -from src.models.boxscore_info import ( - gamebs, quarter, overtime, BoxscoreInfo) -import json -import pandas -from src.models.game_info import ( - gamegame, awayteam, hometeam, broadcast, weather, wind, GameInfo) -from src.models.league_info import ( - game, season, changelog, leagueweek, LeagueInfo) -from src.models.venue_info import (venue1, location, VenueInfo) -import time -import os -import requests -from bson import ObjectId -from mongoengine import DecimalField, EmbeddedDocumentField, Document, StringField, UUIDField, IntField, BooleanField, DateTimeField, EmbeddedDocument, EmbeddedDocumentListField -from uuid import UUID -from datetime import datetime -from flask import Blueprint, jsonify, Flask -import sys -sys.path.append("os.getenv('LPATH')/src/") -if not hasattr(os, 'add_dll_directory'): - def add_dll_directory(path): - pass -bp = Blueprint('current_season_schedule', __name__) - - -def log_and_catch_exceptions(func): - def func_wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except Exception as e: - be_logger.error(f"Error in {func.__name__}: {e}") - raise Exception(f"Error in {func.__name__}: {e}") - return func_wrapper - - -@log_and_catch_exceptions -@bp.route('/fetchAndSaveAllSeasonsSchedule', methods=['GET']) -def fetch_and_save_all_seasons_schedule(): - print("fetch_and_save_all_seasons_schedule called") - API_KEY = os.getenv('APIKEY') - SEASONS_API_URL = "http://api.sportradar.us/nfl/official/trial/v7/en/games/{year}/{season_type}/schedule.json?api_key={API_KEY}" - for year in range(2016, 2024): - for season_type in ['REG', 'PST']: - url = SEASONS_API_URL.format( - year=year, season_type=season_type, API_KEY=API_KEY) - print(datetime.now(), "Requesting URL:", url) - response = requests.get(url) - print("Response status code:", response.status_code) - if response.status_code != 200: - return f"GetCurrentSeasonScheduleError for {year} {season_type}: {response.status_code}" - data = response.json() - venue_info_dict = extract_venue_info(data) - league_info_dict = extract_league_info(data) - game_info_dict = extract_game_info(data) - boxscore_info_dict = extract_boxscore_info(data) - mapped_venues = map_venue_info(venue_info_dict) - mapped_leagues = map_league_info(league_info_dict) - mapped_games = map_game_info(game_info_dict) - mapped_boxscores = map_boxscore_info(boxscore_info_dict) - save_to_database(mapped_venues, mapped_leagues, - mapped_games, mapped_boxscores) - print("Games saved to database") - time.sleep(2) - return "Schedule data for all seasons fetched and saved successfully." - -@log_and_catch_exceptions -@bp.route('/fetchAndSaveWeeklySchedule', methods=['GET']) -def fetch_and_save_weekly_schedule(): - print("fetch_and_save_weekly_schedule called") - API_KEY = os.getenv('APIKEY') - season_year = datetime.now().year - season_type = 'REG' # or 'PRE' or 'POST' depending on the current season - # adjust this calculation as needed - week_number = datetime.now().isocalendar()[1] - 34 - WEEKLY_SCHEDULE_API_URL = 'http://api.sportradar.us/nfl/official/trial/v7/en/games/{season_year}/{season_type}/{week_number}/schedule.json?api_key={API_KEY}' - url = WEEKLY_SCHEDULE_API_URL.format( - season_year=season_year, season_type=season_type, week_number=week_number, API_KEY=API_KEY) - print(datetime.now(), "Requesting URL:", url) - response = requests.get(url) - print("Response status code:", response.status_code) - if response.status_code != 200: - return f"GetCurrentSeasonScheduleError for {season_year} {season_type} {week_number}: {response.status_code}" - data = response.json() - # print("data:", data) - venue_info_dict = extract_venue_info(data) - league_info_dict = extract_league_info(data) - game_info_dict = extract_game_info(data) - boxscore_info_dict = extract_boxscore_info(data) - mapped_venues = map_venue_info(venue_info_dict) - mapped_leagues = map_league_info(league_info_dict) - mapped_games = map_game_info(game_info_dict) - mapped_boxscores = map_boxscore_info(boxscore_info_dict) - save_to_database(mapped_venues, mapped_leagues, - mapped_games, mapped_boxscores) - - time.sleep(2) - return ("Schedule data for all seasons fetched and saved successfully") - - -@log_and_catch_exceptions -def extract_boxscore_info(data): - boxscore_info_dict = {} - weeks_or_week = data.get('weeks') or data.get('week') - if not weeks_or_week: - return boxscore_info_dict - if isinstance(weeks_or_week, dict): - weeks_or_week = [weeks_or_week] - for week in weeks_or_week: - for game in week['games']: - boxscore_info = { - 'game_id': game['id'], - 'attendance': game.get('attendance', None), - } - boxscore_info['home_teambs'] = game['home']['alias'] - boxscore_info['away_teambs'] = game['away']['alias'] - scoring = game.get('scoring', {}) - boxscore_info['home_team_total_points'] = scoring.get( - 'home_points') - boxscore_info['away_team_total_points'] = scoring.get( - 'away_points') - boxscore_info['quarters'] = [] - for quarter in scoring.get('periods', []): - quarter_info = { - 'quarter_id': quarter['id'], - 'quarter_number': quarter['number'], - 'quarter_sequence': quarter['sequence'], - 'away_points_for_quarter': quarter['away_points'], - 'home_points_for_quarter': quarter['home_points'], - } - boxscore_info['quarters'].append(quarter_info) - boxscore_info['overtime'] = [] - for overtime in scoring.get('overtime', []): - overtime_info = { - 'overtime_id': overtime['id'], - 'overtime_number': overtime['number'], - 'overtime_sequence': overtime['sequence'], - 'away_points_for_overtime': overtime['away_points'], - 'home_points_for_overtime': overtime['home_points'], - } - boxscore_info['overtime'].append(overtime_info) - boxscore_info_dict[game['id']] = boxscore_info - return boxscore_info_dict - - -@log_and_catch_exceptions -def map_boxscore_info(boxscore_info_dict): - if boxscore_info_dict is None: - return {} - mapped_boxscores = {} - for gamebs_id in boxscore_info_dict: - try: - gamebs_details = boxscore_info_dict[gamebs_id] - game_embeded = gamebs( - id=gamebs_details['game_id'], - attendance=gamebs_details['attendance'], - hometeam=gamebs_details['home_teambs'], - hometeamtotalpoints=gamebs_details['home_team_total_points'], - awayteam=gamebs_details['away_teambs'], - awayteamtotalpoints=gamebs_details['away_team_total_points'] - ) - quarters_embedded_list = [] - for quarter_data in gamebs_details.get('quarters', []): - quarter_embedded = quarter( - quarter_id=quarter_data['quarter_id'], - quarter_number=quarter_data['quarter_number'], - quarter_sequence=quarter_data['quarter_sequence'], - awayteampointsforquarter=quarter_data['away_points_for_quarter'], - hometeampointsforquarter=quarter_data['home_points_for_quarter'] - ) - quarters_embedded_list.append(quarter_embedded) - - overtime_embedded_list = [] - for overtime_data in gamebs_details.get('overtime', []): - overtime_embedded = overtime( - overtime_id=overtime_data['overtime_id'], - overtime_number=overtime_data['overtime_number'], - overtime_sequence=overtime_data['overtime_sequence'], - awayteamovertimepoints=overtime_data['away_points_for_overtime'], - hometeamovertimepoints=overtime_data['home_points_for_overtime'] - ) - overtime_embedded_list.append(overtime_embedded) - - boxscore_info_instance = BoxscoreInfo( - gamebs=game_embeded, - overtimes=overtime_embedded_list, - quarters=quarters_embedded_list - ) - mapped_boxscores[gamebs_id] = boxscore_info_instance - except Exception as e: - print(f"Error processing gamebs_id {gamebs_id}: {e}") - raise e - # print("Mapped Boxscore Info:", mapped_boxscores) - return mapped_boxscores - - -@log_and_catch_exceptions -def extract_game_info(data): - game_info_dict = {} - weeks_or_week = data.get('weeks') or data.get('week') - if not weeks_or_week: - return game_info_dict - if isinstance(weeks_or_week, dict): - weeks_or_week = [weeks_or_week] # make it a list so we can iterate - for week in weeks_or_week: - for game in week['games']: - game_ginfo = {'season_id': data.get('id', None)} - game_ginfo['week_id'] = week.get('id', None) - game_ginfo['venue_id'] = game['venue']['id'] - game_ginfo['game_id'] = game['id'] - game_ginfo['status'] = game['status'] - game_ginfo['scheduled'] = game['scheduled'] - game_ginfo['attendance'] = game.get( - 'attendance', None) # or some default value - game_ginfo['entry_mode'] = game.get( - 'entry_mode', None) # or some default value - game_ginfo['sr_id'] = game.get('sr_id') - game_ginfo['neutral_site'] = game.get( - 'neutral_site', None) # or some default value - game_ginfo['game_type'] = game.get( - 'game_type', None) # or some default value - game_ginfo['conference_game'] = game.get( - 'conference_game', None) # or some default value - game_ginfo['title'] = game.get( - 'title', None) # or some default value - game_ginfo['duration'] = game.get( - 'duration', None) # or some default value - game_ginfo['home_id'] = game['home']['id'] - game_ginfo['home_name'] = game['home']['name'] - game_ginfo['home_alias'] = game['home']['alias'] - game_ginfo['home_game_number'] = game['home'].get('game_number') - game_ginfo['home_sr_id'] = game['home'].get('sr_id') - game_ginfo['away_id'] = game['away']['id'] - game_ginfo['away_name'] = game['away']['name'] - game_ginfo['away_alias'] = game['away']['alias'] - game_ginfo['away_game_number'] = game['away'].get('game_number') - game_ginfo['away_sr_id'] = game['away'].get('sr_id') - game_ginfo['broadcast_network'] = game.get('broadcast', {}).get( - 'network', None) # or some default value - game_ginfo['broadcast_channel'] = game.get( - 'broadcast', {}).get('channel', None) - game_ginfo['broadcast_satellite'] = game.get( - 'broadcast', {}).get('satellite', None) - game_ginfo['broadcast_internet'] = game.get( - 'broadcast', {}).get('internet', None) - game_ginfo['weather_condition'] = game.get( - 'weather', {}).get('condition', None) - game_ginfo['weather_humidity'] = game.get( - 'weather', {}).get('humidity', None) - game_ginfo['weather_temp'] = game.get( - 'weather', {}).get('temp', None) - game_ginfo['wind_speed'] = game.get( - 'weather', {}).get('wind', {}).get('speed', None) - game_ginfo['wind_direction'] = game.get( - 'weather', {}).get('wind', {}).get('direction', None) - game_info_dict[game['id']] = game_ginfo - # print("game_info_dict:", game_info_dict) - return game_info_dict - - -@log_and_catch_exceptions -def map_game_info(game_info_dict): - mapped_games = {} - for game_id in game_info_dict: - game_details = game_info_dict[game_id] - # print("game_details:", game_details) - # Convert integer IDs to string UUIDs if needed - game_embedded1 = gamegame( - id=game_details['game_id'], - # number=None, - conference_game=game_details['conference_game'], - # coverage=None, - duration=game_details['duration'], - entry_mode=game_details['entry_mode'], - game_type=game_details['game_type'], - sr_id=game_details['sr_id'], - # last_modified=None, - scheduled=game_details['scheduled'], - status=game_details['status'], - title=game_details['title'], - neutral_site=game_details['neutral_site'], - seasonid=game_details['season_id'], - leagueweek=game_details['week_id'], - venueid=game_details['venue_id'] - # season_id=None - ) - away_embedded1 = awayteam( - alias=game_details['away_alias'], - id=game_details['away_id'], - name=game_details['away_name'], - game_number=game_details['away_game_number'], - sr_id=game_details['away_sr_id'], - # market=None - ) - home_embedded1 = hometeam( - alias=game_details['home_alias'], - id=game_details['home_id'], - name=game_details['home_name'], - game_number=game_details['home_game_number'], - sr_id=game_details['home_sr_id'], - # market=None - ) - broadcast_embedded = broadcast( - channel=game_details['broadcast_channel'], - internet=game_details['broadcast_internet'], - network=game_details['broadcast_network'], - satellite=game_details['broadcast_satellite'] - ) - weather_embedded = weather( - condition=game_details['weather_condition'], - humidity=game_details['weather_humidity'], - temp=game_details['weather_temp'] - ) - wind_embedded = wind( - direction=game_details['wind_direction'], - speed=game_details['wind_speed'] - ) - game_info_instance = GameInfo( - gamegame=game_embedded1, - awayteam=away_embedded1, - hometeam=home_embedded1, - broadcast=broadcast_embedded, - weather=weather_embedded, - wind=wind_embedded - ) - mapped_games[game_id] = game_info_instance - # print("Mapped Game Info:", mapped_games) - return mapped_games - - -@log_and_catch_exceptions -def extract_league_info(data): - league_info_dict = {} - season = data # Assuming there's only one season in the list - weeks_or_week = season.get('weeks') or season.get('week') - if not weeks_or_week: - return league_info_dict - if isinstance(weeks_or_week, dict): - weeks_or_week = [weeks_or_week] # make it a list so we can iterate - for week in weeks_or_week: - league_info = { - 'Season Id': season['id'], - 'Season Year': season['year'], - 'Season Type': season['type'], - 'Season Name': season['name'], - 'Week Id': week['id'], - 'Week Sequence': week['sequence'], - 'Week Title': week['title'], - } - bye_week_teams = week.get('bye_week', []) - bye_week_team_info = [] - for team in bye_week_teams: - team_info = team.get('team', {}) # Get the team details - team_id = team_info.get('id') - team_name = team_info.get('name') - team_alias = team_info.get('alias') - # Set a default value if 'sr_id' is missing - team_sr_id = team_info.get('sr_id', None) - bye_week_team_info.append( - {'id': team_id, 'name': team_name, 'alias': team_alias, 'sr_id': team_sr_id}) - league_info['Bye Week Team Info'] = bye_week_team_info - league_info_dict[league_info['Week Id']] = league_info - return league_info_dict - - -@log_and_catch_exceptions -def map_league_info(league_info_dict): - mapped_leagues = {} - for weekid, league_details in league_info_dict.items(): - season_embedded = season( - weekid=league_details['Week Id'], - id=league_details['Season Id'], - name=league_details['Season Name'], - type=league_details['Season Type'], - year=league_details['Season Year'] - ) - week_embedded_list = [] # List to store leagueweek instances - if bye_week_team_info_list := league_details['Bye Week Team Info']: - for team_info in bye_week_team_info_list: - week_embedded = leagueweek( - id=league_details['Week Id'], - sequence=league_details['Week Sequence'], - title=league_details['Week Title'], - byeweekteamalias=team_info.get('alias', None), - byeweekteamid=team_info.get('id', None), - byeweekteamname=team_info.get('name', None), - byeweekteamsrid=team_info.get('sr_id', None) - ) - week_embedded_list.append(week_embedded) - else: - week_embedded = leagueweek( - id=league_details['Week Id'], - sequence=league_details['Week Sequence'], - title=league_details['Week Title'] - ) - week_embedded_list.append(week_embedded) - - # Create a new dictionary for each LeagueInfo instance - league_info_instance = LeagueInfo( - season=season_embedded, - leagueweek=week_embedded_list - ) - mapped_leagues[weekid] = league_info_instance - print("Mapped Leagues:", mapped_leagues) - return mapped_leagues - - -@log_and_catch_exceptions -def extract_venue_info(data): - venue_info_dict = {} - weeks_or_week = data.get('weeks') or data.get('week') - if not weeks_or_week: - return venue_info_dict - if isinstance(weeks_or_week, dict): - weeks_or_week = [weeks_or_week] # make it a list so we can iterate - for week in weeks_or_week: - for game in week['games']: - venue_info = { - 'id': game['venue']['id'], - 'name': game['venue']['name'], - 'city': game['venue']['city'], - 'state': game['venue'].get('state'), - } - venue_info['country'] = game['venue'].get('country') - venue_info['zip'] = game['venue'].get('zip') - venue_info['address'] = game['venue']['address'] - venue_info['capacity'] = game['venue']['capacity'] - venue_info['surface'] = game['venue']['surface'] - venue_info['roof_type'] = game['venue']['roof_type'] - venue_info['sr_id'] = game['venue']['sr_id'] - venue_info['lat'] = game.get('venue', {}).get( - 'location', {}).get('lat', None) - venue_info['lng'] = game.get('venue', {}).get( - 'location', {}).get('lng', None) - venue_info_dict[game['venue']['id']] = venue_info - # print("venue_info_dict:", venue_info_dict) - return venue_info_dict - - -@log_and_catch_exceptions -def map_venue_info(venue_info_dict): - mapped_venues = {} - for venue_id, venue_details in venue_info_dict.items(): - - venue_embedded = venue1( - id=venue_details['id'], - address=venue_details['address'], - capacity=venue_details['capacity'], - city=venue_details['city'], - country=venue_details['country'], - name=venue_details['name'], - sr_id=venue_details['sr_id'], - roof_type=venue_details['roof_type'], - state=venue_details['state'], - surface=venue_details['surface'], - zip=venue_details['zip'] - ) - - location_embedded = location( - lat=venue_details['lat'], - lng=venue_details['lng'] - ) - - venue_info_instance = VenueInfo( - venue1=venue_embedded, - location=location_embedded - ) - mapped_venues[venue_id] = venue_info_instance - # print(len(mapped_venues)) - # print("Mapped_Venues", mapped_venues) - return mapped_venues - -@log_and_catch_exceptions -def save_to_database(mapped_venues, mapped_leagues, mapped_games, mapped_boxscores): - print("save_to_database called") - - def update_collection(model_cls, mapped_data, collection_name): - updated_count = 0 - new_count = 0 - for mapped_entry_info in mapped_data: - mapped_entry = mapped_entry_info - mapped_entry_id = mapped_entry.id # Extract the ID from the mapped entry - existing_entry = model_cls.objects(id=mapped_entry_id).first() - - # Print the ID being checked - print(f"Checking {collection_name} with ID: {mapped_entry_id}") - - if existing_entry: - updated_fields = [] - for field_name in existing_entry._fields.keys(): - existing_value = getattr(existing_entry, field_name) - mapped_value = getattr(mapped_entry, field_name) - if existing_value != mapped_value: - setattr(existing_entry, field_name, mapped_value) - updated_fields.append(field_name) - if updated_fields: - existing_entry.save() - updated_count += 1 - print( - f"Updated {collection_name} {mapped_entry_id}: Updated fields: {', '.join(updated_fields)}") - else: - print( - f"No updates needed for {collection_name} {mapped_entry_id}") - else: - mongo_representation = mapped_entry.to_mongo() - print( - f"Mapped entry: {mapped_entry}, Mongo representation: {mongo_representation}") - new_entry = model_cls(**mapped_entry.to_mongo()) - new_entry.save() - new_count += 1 - print(f"Added new {collection_name} with id {mapped_entry_id}") - - print( - f"Updated {updated_count} {collection_name}s and added {new_count} new {collection_name}s.") - - update_collection(VenueInfo, mapped_venues.values(), "venue") - update_collection(LeagueInfo, mapped_leagues.values(), "league") - update_collection(GameInfo, mapped_games.values(), "game") - update_collection(BoxscoreInfo, mapped_boxscores.values(), "boxscore") \ No newline at end of file +from src.models.boxscore_info import ( + gamebs, quarter, overtime, BoxscoreInfo) +from src.models.game_info import ( + gamegame, awayteam, hometeam, broadcast, weather, wind, GameInfo) +from src.models.league_info import ( + season, leagueweek, LeagueInfo) +from src.models.venue_info import (venue1, location, VenueInfo) +import time +import os +from datetime import datetime +from flask import Blueprint +import sys +from security import safe_requests + +sys.path.append("os.getenv('LPATH')/src/") +if not hasattr(os, 'add_dll_directory'): + def add_dll_directory(path): + pass +bp = Blueprint('current_season_schedule', __name__) + + +def log_and_catch_exceptions(func): + def func_wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as e: + be_logger.error(f"Error in {func.__name__}: {e}") + raise Exception(f"Error in {func.__name__}: {e}") + return func_wrapper + + +@log_and_catch_exceptions +@bp.route('/fetchAndSaveAllSeasonsSchedule', methods=['GET']) +def fetch_and_save_all_seasons_schedule(): + print("fetch_and_save_all_seasons_schedule called") + API_KEY = os.getenv('APIKEY') + SEASONS_API_URL = "http://api.sportradar.us/nfl/official/trial/v7/en/games/{year}/{season_type}/schedule.json?api_key={API_KEY}" + for year in range(2016, 2024): + for season_type in ['REG', 'PST']: + url = SEASONS_API_URL.format( + year=year, season_type=season_type, API_KEY=API_KEY) + print(datetime.now(), "Requesting URL:", url) + response = safe_requests.get(url) + print("Response status code:", response.status_code) + if response.status_code != 200: + return f"GetCurrentSeasonScheduleError for {year} {season_type}: {response.status_code}" + data = response.json() + venue_info_dict = extract_venue_info(data) + league_info_dict = extract_league_info(data) + game_info_dict = extract_game_info(data) + boxscore_info_dict = extract_boxscore_info(data) + mapped_venues = map_venue_info(venue_info_dict) + mapped_leagues = map_league_info(league_info_dict) + mapped_games = map_game_info(game_info_dict) + mapped_boxscores = map_boxscore_info(boxscore_info_dict) + save_to_database(mapped_venues, mapped_leagues, + mapped_games, mapped_boxscores) + print("Games saved to database") + time.sleep(2) + return "Schedule data for all seasons fetched and saved successfully." + +@log_and_catch_exceptions +@bp.route('/fetchAndSaveWeeklySchedule', methods=['GET']) +def fetch_and_save_weekly_schedule(): + print("fetch_and_save_weekly_schedule called") + API_KEY = os.getenv('APIKEY') + season_year = datetime.now().year + season_type = 'REG' # or 'PRE' or 'POST' depending on the current season + # adjust this calculation as needed + week_number = datetime.now().isocalendar()[1] - 34 + WEEKLY_SCHEDULE_API_URL = 'http://api.sportradar.us/nfl/official/trial/v7/en/games/{season_year}/{season_type}/{week_number}/schedule.json?api_key={API_KEY}' + url = WEEKLY_SCHEDULE_API_URL.format( + season_year=season_year, season_type=season_type, week_number=week_number, API_KEY=API_KEY) + print(datetime.now(), "Requesting URL:", url) + response = safe_requests.get(url) + print("Response status code:", response.status_code) + if response.status_code != 200: + return f"GetCurrentSeasonScheduleError for {season_year} {season_type} {week_number}: {response.status_code}" + data = response.json() + # print("data:", data) + venue_info_dict = extract_venue_info(data) + league_info_dict = extract_league_info(data) + game_info_dict = extract_game_info(data) + boxscore_info_dict = extract_boxscore_info(data) + mapped_venues = map_venue_info(venue_info_dict) + mapped_leagues = map_league_info(league_info_dict) + mapped_games = map_game_info(game_info_dict) + mapped_boxscores = map_boxscore_info(boxscore_info_dict) + save_to_database(mapped_venues, mapped_leagues, + mapped_games, mapped_boxscores) + + time.sleep(2) + return ("Schedule data for all seasons fetched and saved successfully") + + +@log_and_catch_exceptions +def extract_boxscore_info(data): + boxscore_info_dict = {} + weeks_or_week = data.get('weeks') or data.get('week') + if not weeks_or_week: + return boxscore_info_dict + if isinstance(weeks_or_week, dict): + weeks_or_week = [weeks_or_week] + for week in weeks_or_week: + for game in week['games']: + boxscore_info = { + 'game_id': game['id'], + 'attendance': game.get('attendance', None), + } + boxscore_info['home_teambs'] = game['home']['alias'] + boxscore_info['away_teambs'] = game['away']['alias'] + scoring = game.get('scoring', {}) + boxscore_info['home_team_total_points'] = scoring.get( + 'home_points') + boxscore_info['away_team_total_points'] = scoring.get( + 'away_points') + boxscore_info['quarters'] = [] + for quarter in scoring.get('periods', []): + quarter_info = { + 'quarter_id': quarter['id'], + 'quarter_number': quarter['number'], + 'quarter_sequence': quarter['sequence'], + 'away_points_for_quarter': quarter['away_points'], + 'home_points_for_quarter': quarter['home_points'], + } + boxscore_info['quarters'].append(quarter_info) + boxscore_info['overtime'] = [] + for overtime in scoring.get('overtime', []): + overtime_info = { + 'overtime_id': overtime['id'], + 'overtime_number': overtime['number'], + 'overtime_sequence': overtime['sequence'], + 'away_points_for_overtime': overtime['away_points'], + 'home_points_for_overtime': overtime['home_points'], + } + boxscore_info['overtime'].append(overtime_info) + boxscore_info_dict[game['id']] = boxscore_info + return boxscore_info_dict + + +@log_and_catch_exceptions +def map_boxscore_info(boxscore_info_dict): + if boxscore_info_dict is None: + return {} + mapped_boxscores = {} + for gamebs_id in boxscore_info_dict: + try: + gamebs_details = boxscore_info_dict[gamebs_id] + game_embeded = gamebs( + id=gamebs_details['game_id'], + attendance=gamebs_details['attendance'], + hometeam=gamebs_details['home_teambs'], + hometeamtotalpoints=gamebs_details['home_team_total_points'], + awayteam=gamebs_details['away_teambs'], + awayteamtotalpoints=gamebs_details['away_team_total_points'] + ) + quarters_embedded_list = [] + for quarter_data in gamebs_details.get('quarters', []): + quarter_embedded = quarter( + quarter_id=quarter_data['quarter_id'], + quarter_number=quarter_data['quarter_number'], + quarter_sequence=quarter_data['quarter_sequence'], + awayteampointsforquarter=quarter_data['away_points_for_quarter'], + hometeampointsforquarter=quarter_data['home_points_for_quarter'] + ) + quarters_embedded_list.append(quarter_embedded) + + overtime_embedded_list = [] + for overtime_data in gamebs_details.get('overtime', []): + overtime_embedded = overtime( + overtime_id=overtime_data['overtime_id'], + overtime_number=overtime_data['overtime_number'], + overtime_sequence=overtime_data['overtime_sequence'], + awayteamovertimepoints=overtime_data['away_points_for_overtime'], + hometeamovertimepoints=overtime_data['home_points_for_overtime'] + ) + overtime_embedded_list.append(overtime_embedded) + + boxscore_info_instance = BoxscoreInfo( + gamebs=game_embeded, + overtimes=overtime_embedded_list, + quarters=quarters_embedded_list + ) + mapped_boxscores[gamebs_id] = boxscore_info_instance + except Exception as e: + print(f"Error processing gamebs_id {gamebs_id}: {e}") + raise e + # print("Mapped Boxscore Info:", mapped_boxscores) + return mapped_boxscores + + +@log_and_catch_exceptions +def extract_game_info(data): + game_info_dict = {} + weeks_or_week = data.get('weeks') or data.get('week') + if not weeks_or_week: + return game_info_dict + if isinstance(weeks_or_week, dict): + weeks_or_week = [weeks_or_week] # make it a list so we can iterate + for week in weeks_or_week: + for game in week['games']: + game_ginfo = {'season_id': data.get('id', None)} + game_ginfo['week_id'] = week.get('id', None) + game_ginfo['venue_id'] = game['venue']['id'] + game_ginfo['game_id'] = game['id'] + game_ginfo['status'] = game['status'] + game_ginfo['scheduled'] = game['scheduled'] + game_ginfo['attendance'] = game.get( + 'attendance', None) # or some default value + game_ginfo['entry_mode'] = game.get( + 'entry_mode', None) # or some default value + game_ginfo['sr_id'] = game.get('sr_id') + game_ginfo['neutral_site'] = game.get( + 'neutral_site', None) # or some default value + game_ginfo['game_type'] = game.get( + 'game_type', None) # or some default value + game_ginfo['conference_game'] = game.get( + 'conference_game', None) # or some default value + game_ginfo['title'] = game.get( + 'title', None) # or some default value + game_ginfo['duration'] = game.get( + 'duration', None) # or some default value + game_ginfo['home_id'] = game['home']['id'] + game_ginfo['home_name'] = game['home']['name'] + game_ginfo['home_alias'] = game['home']['alias'] + game_ginfo['home_game_number'] = game['home'].get('game_number') + game_ginfo['home_sr_id'] = game['home'].get('sr_id') + game_ginfo['away_id'] = game['away']['id'] + game_ginfo['away_name'] = game['away']['name'] + game_ginfo['away_alias'] = game['away']['alias'] + game_ginfo['away_game_number'] = game['away'].get('game_number') + game_ginfo['away_sr_id'] = game['away'].get('sr_id') + game_ginfo['broadcast_network'] = game.get('broadcast', {}).get( + 'network', None) # or some default value + game_ginfo['broadcast_channel'] = game.get( + 'broadcast', {}).get('channel', None) + game_ginfo['broadcast_satellite'] = game.get( + 'broadcast', {}).get('satellite', None) + game_ginfo['broadcast_internet'] = game.get( + 'broadcast', {}).get('internet', None) + game_ginfo['weather_condition'] = game.get( + 'weather', {}).get('condition', None) + game_ginfo['weather_humidity'] = game.get( + 'weather', {}).get('humidity', None) + game_ginfo['weather_temp'] = game.get( + 'weather', {}).get('temp', None) + game_ginfo['wind_speed'] = game.get( + 'weather', {}).get('wind', {}).get('speed', None) + game_ginfo['wind_direction'] = game.get( + 'weather', {}).get('wind', {}).get('direction', None) + game_info_dict[game['id']] = game_ginfo + # print("game_info_dict:", game_info_dict) + return game_info_dict + + +@log_and_catch_exceptions +def map_game_info(game_info_dict): + mapped_games = {} + for game_id in game_info_dict: + game_details = game_info_dict[game_id] + # print("game_details:", game_details) + # Convert integer IDs to string UUIDs if needed + game_embedded1 = gamegame( + id=game_details['game_id'], + # number=None, + conference_game=game_details['conference_game'], + # coverage=None, + duration=game_details['duration'], + entry_mode=game_details['entry_mode'], + game_type=game_details['game_type'], + sr_id=game_details['sr_id'], + # last_modified=None, + scheduled=game_details['scheduled'], + status=game_details['status'], + title=game_details['title'], + neutral_site=game_details['neutral_site'], + seasonid=game_details['season_id'], + leagueweek=game_details['week_id'], + venueid=game_details['venue_id'] + # season_id=None + ) + away_embedded1 = awayteam( + alias=game_details['away_alias'], + id=game_details['away_id'], + name=game_details['away_name'], + game_number=game_details['away_game_number'], + sr_id=game_details['away_sr_id'], + # market=None + ) + home_embedded1 = hometeam( + alias=game_details['home_alias'], + id=game_details['home_id'], + name=game_details['home_name'], + game_number=game_details['home_game_number'], + sr_id=game_details['home_sr_id'], + # market=None + ) + broadcast_embedded = broadcast( + channel=game_details['broadcast_channel'], + internet=game_details['broadcast_internet'], + network=game_details['broadcast_network'], + satellite=game_details['broadcast_satellite'] + ) + weather_embedded = weather( + condition=game_details['weather_condition'], + humidity=game_details['weather_humidity'], + temp=game_details['weather_temp'] + ) + wind_embedded = wind( + direction=game_details['wind_direction'], + speed=game_details['wind_speed'] + ) + game_info_instance = GameInfo( + gamegame=game_embedded1, + awayteam=away_embedded1, + hometeam=home_embedded1, + broadcast=broadcast_embedded, + weather=weather_embedded, + wind=wind_embedded + ) + mapped_games[game_id] = game_info_instance + # print("Mapped Game Info:", mapped_games) + return mapped_games + + +@log_and_catch_exceptions +def extract_league_info(data): + league_info_dict = {} + season = data # Assuming there's only one season in the list + weeks_or_week = season.get('weeks') or season.get('week') + if not weeks_or_week: + return league_info_dict + if isinstance(weeks_or_week, dict): + weeks_or_week = [weeks_or_week] # make it a list so we can iterate + for week in weeks_or_week: + league_info = { + 'Season Id': season['id'], + 'Season Year': season['year'], + 'Season Type': season['type'], + 'Season Name': season['name'], + 'Week Id': week['id'], + 'Week Sequence': week['sequence'], + 'Week Title': week['title'], + } + bye_week_teams = week.get('bye_week', []) + bye_week_team_info = [] + for team in bye_week_teams: + team_info = team.get('team', {}) # Get the team details + team_id = team_info.get('id') + team_name = team_info.get('name') + team_alias = team_info.get('alias') + # Set a default value if 'sr_id' is missing + team_sr_id = team_info.get('sr_id', None) + bye_week_team_info.append( + {'id': team_id, 'name': team_name, 'alias': team_alias, 'sr_id': team_sr_id}) + league_info['Bye Week Team Info'] = bye_week_team_info + league_info_dict[league_info['Week Id']] = league_info + return league_info_dict + + +@log_and_catch_exceptions +def map_league_info(league_info_dict): + mapped_leagues = {} + for weekid, league_details in league_info_dict.items(): + season_embedded = season( + weekid=league_details['Week Id'], + id=league_details['Season Id'], + name=league_details['Season Name'], + type=league_details['Season Type'], + year=league_details['Season Year'] + ) + week_embedded_list = [] # List to store leagueweek instances + if bye_week_team_info_list := league_details['Bye Week Team Info']: + for team_info in bye_week_team_info_list: + week_embedded = leagueweek( + id=league_details['Week Id'], + sequence=league_details['Week Sequence'], + title=league_details['Week Title'], + byeweekteamalias=team_info.get('alias', None), + byeweekteamid=team_info.get('id', None), + byeweekteamname=team_info.get('name', None), + byeweekteamsrid=team_info.get('sr_id', None) + ) + week_embedded_list.append(week_embedded) + else: + week_embedded = leagueweek( + id=league_details['Week Id'], + sequence=league_details['Week Sequence'], + title=league_details['Week Title'] + ) + week_embedded_list.append(week_embedded) + + # Create a new dictionary for each LeagueInfo instance + league_info_instance = LeagueInfo( + season=season_embedded, + leagueweek=week_embedded_list + ) + mapped_leagues[weekid] = league_info_instance + print("Mapped Leagues:", mapped_leagues) + return mapped_leagues + + +@log_and_catch_exceptions +def extract_venue_info(data): + venue_info_dict = {} + weeks_or_week = data.get('weeks') or data.get('week') + if not weeks_or_week: + return venue_info_dict + if isinstance(weeks_or_week, dict): + weeks_or_week = [weeks_or_week] # make it a list so we can iterate + for week in weeks_or_week: + for game in week['games']: + venue_info = { + 'id': game['venue']['id'], + 'name': game['venue']['name'], + 'city': game['venue']['city'], + 'state': game['venue'].get('state'), + } + venue_info['country'] = game['venue'].get('country') + venue_info['zip'] = game['venue'].get('zip') + venue_info['address'] = game['venue']['address'] + venue_info['capacity'] = game['venue']['capacity'] + venue_info['surface'] = game['venue']['surface'] + venue_info['roof_type'] = game['venue']['roof_type'] + venue_info['sr_id'] = game['venue']['sr_id'] + venue_info['lat'] = game.get('venue', {}).get( + 'location', {}).get('lat', None) + venue_info['lng'] = game.get('venue', {}).get( + 'location', {}).get('lng', None) + venue_info_dict[game['venue']['id']] = venue_info + # print("venue_info_dict:", venue_info_dict) + return venue_info_dict + + +@log_and_catch_exceptions +def map_venue_info(venue_info_dict): + mapped_venues = {} + for venue_id, venue_details in venue_info_dict.items(): + + venue_embedded = venue1( + id=venue_details['id'], + address=venue_details['address'], + capacity=venue_details['capacity'], + city=venue_details['city'], + country=venue_details['country'], + name=venue_details['name'], + sr_id=venue_details['sr_id'], + roof_type=venue_details['roof_type'], + state=venue_details['state'], + surface=venue_details['surface'], + zip=venue_details['zip'] + ) + + location_embedded = location( + lat=venue_details['lat'], + lng=venue_details['lng'] + ) + + venue_info_instance = VenueInfo( + venue1=venue_embedded, + location=location_embedded + ) + mapped_venues[venue_id] = venue_info_instance + # print(len(mapped_venues)) + # print("Mapped_Venues", mapped_venues) + return mapped_venues + +@log_and_catch_exceptions +def save_to_database(mapped_venues, mapped_leagues, mapped_games, mapped_boxscores): + print("save_to_database called") + + def update_collection(model_cls, mapped_data, collection_name): + updated_count = 0 + new_count = 0 + for mapped_entry_info in mapped_data: + mapped_entry = mapped_entry_info + mapped_entry_id = mapped_entry.id # Extract the ID from the mapped entry + existing_entry = model_cls.objects(id=mapped_entry_id).first() + + # Print the ID being checked + print(f"Checking {collection_name} with ID: {mapped_entry_id}") + + if existing_entry: + updated_fields = [] + for field_name in existing_entry._fields.keys(): + existing_value = getattr(existing_entry, field_name) + mapped_value = getattr(mapped_entry, field_name) + if existing_value != mapped_value: + setattr(existing_entry, field_name, mapped_value) + updated_fields.append(field_name) + if updated_fields: + existing_entry.save() + updated_count += 1 + print( + f"Updated {collection_name} {mapped_entry_id}: Updated fields: {', '.join(updated_fields)}") + else: + print( + f"No updates needed for {collection_name} {mapped_entry_id}") + else: + mongo_representation = mapped_entry.to_mongo() + print( + f"Mapped entry: {mapped_entry}, Mongo representation: {mongo_representation}") + new_entry = model_cls(**mapped_entry.to_mongo()) + new_entry.save() + new_count += 1 + print(f"Added new {collection_name} with id {mapped_entry_id}") + + print( + f"Updated {updated_count} {collection_name}s and added {new_count} new {collection_name}s.") + + update_collection(VenueInfo, mapped_venues.values(), "venue") + update_collection(LeagueInfo, mapped_leagues.values(), "league") + update_collection(GameInfo, mapped_games.values(), "game") + update_collection(BoxscoreInfo, mapped_boxscores.values(), "boxscore") diff --git a/backend-container/src/sportsradar/extract/additionalfeeds.py b/backend-container/src/sportsradar/extract/additionalfeeds.py index faf957d2..5498ffa0 100644 --- a/backend-container/src/sportsradar/extract/additionalfeeds.py +++ b/backend-container/src/sportsradar/extract/additionalfeeds.py @@ -50,3 +50,172 @@ def get_weekly_depth_charts( ) logger.info("Data retrieved successfully.") return result + + def get_daily_change_log( + self, + access_level, + version, + language_code, + year, + month, + day, + file_format, + api_key, + ): + """ + :param: access_level: + :param: version: + :param: language_code: + :param: year: + :month: month: + :param day: + :param format: + :param api_key: + """ + if not api_key: + logger.error("API key not found in environment variables") + raise ValueError("API key not found in environemt variables") + datastore = DataStore(SportsRadarFetcher()) + result = datastore.fetch_data( + url=f"{self.base_url}/{access_level}/{version}/{language_code}/league/{year}/{month}/{day}/changes.{file_format}?api_key={api_key}" + ) + logger.info("Data Retrieved successfully.") + return result + + def get_daily_transactions( + self, + access_level, + version, + language_code, + year, + month, + day, + file_format, + api_key, + ): + """ + :param: access_level: + :param: version: + :param: language_code: + :param: year: + :month: month: + :param day: + :param format: + :param api_key: + """ + if not api_key: + logger.error("API key not found in environment variables") + raise ValueError("API key not found in environemt variables") + datastore = DataStore(SportsRadarFetcher()) + result = datastore.fetch_data( + url=f"{self.base_url}/{access_level}/{version}/{language_code}/league/{year}/{month}/{day}/transactions.{file_format}?api_key={api_key}" + ) + logger.info("Data Retrieved successfully.") + return result + + def get_league_hierarchy( + self, access_level, version, language_code, file_format, api_key + ): + """ + Get the league_hierarchy + :param: access_level: + :param: version: + :param: language_code: + :param format: + :param api_key: + + return: The league hierarchy + """ + if not api_key: + logger.error("API key not found in environment variables") + raise ValueError("API key not found in environemt variables") + datastore = DataStore(SportsRadarFetcher()) + result = datastore.fetch_data( + url=f"{self.base_url}/{access_level}/{version}/{language_code}/league/hierarchy.{file_format}?api_key={api_key}" + ) + logger.info("Data Retrieved successfully.") + return result + + def get_postgame_standings( + self, + access_level, + version, + language_code, + year, + nfl_season, + file_format, + api_key, + ): + """ + Get the postgame standings for a given year + :param access_level: + :param version: + :param language_code: + :param year: Year in 4 digit format (YYYY). + :param nfl_season: Preseason (PRE), Regular Season (REG), or Post-Season (PST). + :param file_format: + :param api_key: + :return: The postgame standings for the given year, nfl_season + """ + if not api_key: + logger.error("API key not found in environment variables") + raise ValueError("API key not found in environemt variables") + datastore = DataStore(SportsRadarFetcher()) + result = datastore.fetch_data( + url=f"{self.base_url}/{access_level}/{version}/{language_code}/seasons/{year}/{nfl_season}/standings/season.{file_format}?api_key={api_key}" + ) + logger.info("Data Retrieved successfully.") + return result + + def get_seasons(self, access_level, version, language_code, file_format, api_key): + """ + Get the seasons for a given team_id + :param access_level: + :param version: + :param language_code: + :param file_format: + :param api_key: + :return: The seasons data + """ + if not api_key: + logger.error("API key not found in environment variables") + raise ValueError("API key not found in environemt variables") + datastore = DataStore(SportsRadarFetcher()) + result = datastore.fetch_data( + url=f"{self.base_url}/{access_level}/{version}/{language_code}/league/seasons.{file_format}?api_key={api_key}" + ) + logger.info("Data Retrieved successfully.") + return result + + def get_weekly_injuries( + self, + access_level, + version, + language_code, + year, + nfl_season, + nfl_season_week, + file_format, + api_key, + ): + """ + Get the weekly injuries + :param access_level: + :param version: + :param language_code: + :param year: Year in 4 digit format (YYYY). + :param nfl_season: Preseason (PRE), Regular Season (REG), or Post-Season (PST). + :param nfl_season_week: The number of weeks into the season in 2 digit format (WW). + :param file_format: + :param api_key: + :return: The weeekly injuries for the given year, nfl_season, nfl_season_week + """ + if not api_key: + logger.error("API key not found in environment variables.") + raise ValueError("API key not found in environment variables") + datastore = DataStore(SportsRadarFetcher()) + result = datastore.fetch_data( + url=f"{self.base_url}/{access_level}/{version}/{language_code}/seasons/{year}/{nfl_season}/{nfl_season_week}/injuries.{file_format}?api_key={api_key}" + ) + logger.info("Data retrieved successfully.") + return result diff --git a/backend-container/src/sportsradar/extract/draftfeeds.py b/backend-container/src/sportsradar/extract/draftfeeds.py new file mode 100644 index 00000000..c9a85df9 --- /dev/null +++ b/backend-container/src/sportsradar/extract/draftfeeds.py @@ -0,0 +1,135 @@ +from dotenv import load_dotenv +from src.sportsradar import logging_helpers +from src.sportsradar.workspace.datastore import DataStore, SportsRadarFetcher + +load_dotenv("../../../.env") + +logger = logging_helpers.get_logger(__name__) + + +class DraftsFeeds: + """This class is reponsible for extraction of draft feeds from SportRadar.""" + + def __init__(self, base_url): + """ + Initialize an instance of the class. + :param base_url: The base URL for the API. + :type base_url: str + """ + self.base_url = base_url + + def get_draft_summary( + self, access_level, version, language_code, year, file_format, api_key + ): + """ + Get the depth_charts for a given team_id + :param access_level: + :param version: + :param language_code: + :param year: Year in 4 digit format (YYYY). + :param file_format: + :param api_key: + :return: The draft_summary for the given year + """ + if not api_key: + logger.error("API key not found in environemnt variables.") + raise ValueError("API key not found in environment variables") + datastore = DataStore(SportsRadarFetcher()) + result = datastore.fetch_data( + url=f"{self.base_url}/{access_level}/{version}/{language_code}/{year}/draft.{file_format}?api_key={api_key}" + ) + logger.info("Data retrieved successfully") + return result + + def get_prospects( + self, access_level, version, language_code, year, file_format, api_key + ): + """ + Get the prospects for a given year + :param access_level: + :param version: + :param language_code: + :param year: Year in 4 digit format (YYYY). + :param file_format: + :param api_key: + :return: The prospects for the given year + """ + if not api_key: + logger.error("API key not found in environment variables") + raise ValueError("API key not found in environemt variables") + datastore = DataStore(SportsRadarFetcher()) + result = datastore.fetch_data( + url=f"{self.base_url}/{access_level}/{version}/{language_code}/{year}/prospects.{file_format}?api_key={api_key}" + ) + logger.info("Data Retrieved successfully.") + return result + + def get_team_draft_summary( + self, access_level, version, language_code, year, team_id, file_format, api_key + ): + """ " + Get the team_draft_summary for a given team_id + :param access_level: + :param version: + :param language_code: + :param year: Year in 4 digit format (YYYY). + :param team_id: + :param file_format: + :param api_key: + :return: The team draft summary for the given team year, + """ + if not api_key: + logger.error("API key not found in environment variables") + raise ValueError("API key not found in environemt variables") + datastore = DataStore(SportsRadarFetcher()) + result = datastore.fetch_data( + url=f"{self.base_url}/{access_level}/{version}/{language_code}/{year}/teams/{team_id}/draft.{file_format}?api_key={api_key}" + ) + logger.info("Data Retrieved successfully.") + return result + + def get_top_prospects( + self, access_level, version, language_code, year, file_format, api_key + ): + """ " + Get the top_prospects for a given team_id + :param access_level: + :param version: + :param language_code: + :param year: Year in 4 digit format (YYYY). + :param file_format: + :param api_key: + :return: The team top prospects for the given year + """ + if not api_key: + logger.error("API key not found in environment variables") + raise ValueError("API key not found in environemt variables") + datastore = DataStore(SportsRadarFetcher()) + result = datastore.fetch_data( + url=f"{self.base_url}/{access_level}/{version}/{language_code}/{year}/top_prospects/.{file_format}?api_key={api_key}" + ) + logger.info("Data Retrieved successfully.") + return result + + def get_trades( + self, access_level, version, language_code, year, file_format, api_key + ): + """ " + Get the top_prospects for a given team_id + :param access_level: + :param version: + :param language_code: + :param year: Year in 4 digit format (YYYY). + :param file_format: + :param api_key: + :return: The team top prospects for the given year + """ + if not api_key: + logger.error("API key not found in environment variables") + raise ValueError("API key not found in environemt variables") + datastore = DataStore(SportsRadarFetcher()) + result = datastore.fetch_data( + url=f"{self.base_url}/{access_level}/{version}/{language_code}/{year}/trades/.{file_format}?api_key={api_key}" + ) + logger.info("Data Retrieved successfully.") + return result diff --git a/backend-container/src/sportsradar/extract/primaryfeeds.py b/backend-container/src/sportsradar/extract/primaryfeeds.py index c6343aab..19e61f5e 100644 --- a/backend-container/src/sportsradar/extract/primaryfeeds.py +++ b/backend-container/src/sportsradar/extract/primaryfeeds.py @@ -89,9 +89,10 @@ def get_seasons_schedule( logger.error("API key not found in environment variable.") raise ValueError("API key not found in environment variables") datastore = DataStore(SportsRadarFetcher()) - return datastore.fetch_data( + result = datastore.fetch_data( url=f"{self.base_url}/{access_level}/{version}/{language_code}/games/{year}/{season_type}/schedule.{file_format}?api_key={api_key}" ) + return result def get_weekly_schedule( self, @@ -122,6 +123,8 @@ def get_weekly_schedule( logger.error("API key not found in enviroment variable.") raise ValueError("API key not found in environment variables") datastore = DataStore(SportsRadarFetcher()) - return datastore.fetch_data( + result = datastore.fetch_data( url=f"{self.base_url}/{access_level}/{version}/{language_code}/games/{season_year}/{season_type}/{week_number}/schedule.{file_format}?api_key={api_key}" ) + + return result diff --git a/backend-container/src/sportsradar/extract/teamfeeds.py b/backend-container/src/sportsradar/extract/teamfeeds.py index 4b46d3ea..57664c6b 100644 --- a/backend-container/src/sportsradar/extract/teamfeeds.py +++ b/backend-container/src/sportsradar/extract/teamfeeds.py @@ -73,3 +73,26 @@ def get_seasonal_statistics( ) logger.info("Data retrieved successfully.") return result + + def get_team_profile( + self, access_level, version, language_code, team_id, file_format, api_key + ): + """ + Get the team_profile for a given team_id + :param access_level: + :param version: + :param language_code: + :param team_id: + :param file_format: + :param api_key: + :return: The team_profile feed of the given year + """ + if not api_key: + logger.error("API key not found in environment variables.") + raise ValueError("API key not found in environment variables") + datastore = DataStore(SportsRadarFetcher()) + result = datastore.fetch_data( + url=f"{self.base_url}/{access_level}/{version}/{language_code}/teams/{team_id}/profile.{file_format}?api_key={api_key}" + ) + logger.info("Data retrieved successfully.") + return result diff --git a/backend-container/src/sportsradar/simulation/__init__.py b/backend-container/src/sportsradar/simulation/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend-container/src/sportsradar/simulation/available_recordings.py b/backend-container/src/sportsradar/simulation/available_recordings.py new file mode 100644 index 00000000..d05592e3 --- /dev/null +++ b/backend-container/src/sportsradar/simulation/available_recordings.py @@ -0,0 +1,95 @@ +from dotenv import load_dotenv +import requests +from src.sportsradar import logging_helpers + + +load_dotenv("../../../.env") +logger = logging_helpers.get_logger(__name__) + + +class AvailableRecordings: + """ + Module containing the `AvailableRecordings` class for fetching recordings from an API. + + :class:`AvailableRecordings`: + This class provides methods to interact with an API for fetching available recordings. + + :ivar base_url: The base URL for the API. + + :ivar NFL_LEAGUE: Class attribute representing the NFL league. + + :method:`__init__`: + Initializes the instance of `AvailableRecordings`. + + :param base_url: The base URL for the API. + + :method:`construct_query`: + Formulates the GraphQL query for fetching recordings and returns it. + + :return: The GraphQL query string for fetching recordings. + + :method:`post_json_data`: + Sends a POST request with JSON data and GraphQL query. + + :param query: The GraphQL query to be sent in the request. + :param league: The specific league to fetch. Default is `NFL_LEAGUE`. + """ + + NFL_LEAGUE = "nfl" + + def __init__(self, base_url): + """ + Initialize the instance of AvailableRecordings. + :param base_url: The base URL for the API. + """ + self.base_url = base_url + + def construct_query(self): + """ + Formulate the GraphQL query for fetching recordings and return it. + :return: The GraphQL query string for fetching recordings. + """ + return """ + query getRecordings($league: String){ + recordings(league: $league){ + id + scheduled + meta + league + start + end + title + apis { + name + description + formats + } + } + } + """ + + def post_json_data(self, query, league=NFL_LEAGUE): + """ + Send a POST request with JSON data and GraphQL query. + :param query: The GraphQL query to be sent in the request + :param league: The specific league to fetch + """ + headers = {"Content-Type": "application/json"} + json_data = {"query": query, "variables": {"league": league}} + + try: + response = requests.post(self.base_url, headers=headers, json=json_data) + logger.info("Data retrieved successfully.") + return response + except requests.exceptions.RequestException as err: + logger.error(f"Error: {err}") + return None + + +# url = 'https://playback.sportradar.com/graphql' +# rec = AvailableRecordings(url) +# query = rec.construct_query() +# data = rec.post_json_data(query) +# print(type(data)) +# save_data(response=data, database=os.environ.get("MONGODB_DATABASE"), collection='test_simulation_boxscore',db_uri=os.environ.get("MONGODB_URL")) +# save_data(response=data, database='sportradar', collection='test_simulation_boxscore',db_uri='mongodb://localhost:27017') diff --git a/backend-container/src/sportsradar/simulation/config.py b/backend-container/src/sportsradar/simulation/config.py new file mode 100644 index 00000000..a6cd6d5e --- /dev/null +++ b/backend-container/src/sportsradar/simulation/config.py @@ -0,0 +1,13 @@ +import os +from dotenv import load_dotenv + +load_dotenv("../../../.env") + + +class Config: + """Class to handle configuration settings""" + + BASE_URL = "https://playback.sportradar.com" + CONTENT_TYPE = "json" + LEAGUE = "nfl" + MONGODB_URL = f"{os.environ.get('MONGODB_URL')}" diff --git a/backend-container/src/sportsradar/simulation/gamefeeds.py b/backend-container/src/sportsradar/simulation/gamefeeds.py new file mode 100644 index 00000000..5b747813 --- /dev/null +++ b/backend-container/src/sportsradar/simulation/gamefeeds.py @@ -0,0 +1,111 @@ +import requests +from src.sportsradar.simulation.available_recordings import AvailableRecordings +from src.sportsradar.simulation.session import create_session +from src.sportsradar.simulation.config import Config + +GAME_FEEDS_TYPE = "replay" + + +class GameFeeds: + """ + This class is used to interact with game feeds. + + Attributes: + base_url (str): The base URL for the game feeds. + game_feeds (str): The type of game feeds. + + Methods: + get_available_recordings: Retrieve the available recordings. + get_session: Get the session for a given recording ID. + """ + + def __init__(self): + self.base_url = Config.BASE_URL + self.game_feeds = "replay" + + def get_available_recordings(self): + av_rec = AvailableRecordings(base_url=f"{self.base_url}/graphql") + query = av_rec.construct_query() + recording_id = av_rec.post_json_data(query) + return recording_id.json()["data"]["recordings"][0]["id"] + + def get_session(self, recording_id): + session = create_session( + url=f"{self.base_url}/graphql", recording_id=recording_id + ) + return session.json()["data"]["createSession"] + + +def get_game_boxscore(recording_id, session_id): + """ + :param recording_id: The unique identifier for the recording of the game. + :param session_id: The unique identifier for the session of the game. + :return: The JSON response containing the game boxscore data. + + This method retrieves the game boxscore data for a given recording and session of a game. It constructs a URL using the base URL, game feeds type, league, recording id, content type + *, and session id. The constructed URL is then used to make a GET request to the specified endpoint. If the response status code is not 200, an exception is raised with the corresponding + * status code. The JSON response is returned as the result of the method. + """ + url = f"{Config.BASE_URL}/{GAME_FEEDS_TYPE}/{Config.LEAGUE}/{recording_id}?feed=boxscore&contentType={Config.CONTENT_TYPE}&sessionId={session_id}" + response = requests.get(url=url) + if response.status_code != 200: + raise Exception(f"Request failed with status code: {response.status_code}") + return response.json() + + +def get_game_info(recording_id, session_id): + """ + + :param recording_id: The ID of the recording. + :param session_id: The ID of the session. + :return: The game information in JSON format. + + """ + url = f"{Config.BASE_URL}/{GAME_FEEDS_TYPE}/{Config.LEAGUE}/{recording_id}?feed=game&contentType={Config.CONTENT_TYPE}&sessionId={session_id}" + response = requests.get(url=url) + if response.status_code != 200: + raise Exception(f"Request failed with status code: {response.status_code}") + return response.json() + + +def get_pbp_info(recording_id, session_id): + """ + Get play-by-play information for a recording. + + :param recording_id: The ID of the recording. + :param session_id: The session ID. + :return: The play-by-play information in JSON format. + """ + url = f"{Config.BASE_URL}/{GAME_FEEDS_TYPE}/{Config.LEAGUE}/{recording_id}?feed=pbp&contentType={Config.CONTENT_TYPE}&sessionId={session_id}" + response = requests.get(url=url) + if response.status_code != 200: + raise Exception(f"Request failed with status code: {response.status_code}") + return response.json() + + +def get_game_roster(recording_id, session_id): + """ + :param recording_id: Recording ID of the game. + :param session_id: Session ID of the game. + :return: JSON object containing the game roster. + + This method retrieves the game roster for a specific game identified by the recording ID and session ID. It makes a GET request to the specified URL, which includes the recording ID + *, session ID, and other necessary parameters. If the request is successful (status code 200), the response is returned as a JSON object. If the request fails, an exception is raised + * with the corresponding status code. + """ + url = f"{Config.BASE_URL}/{GAME_FEEDS_TYPE}/{Config.LEAGUE}/{recording_id}?feed=rosters&contentType={Config.CONTENT_TYPE}&sessionId={session_id}" + response = requests.get(url=url) + print(response) + if response.status_code != 200: + raise Exception(f"Request failed with status code: {response.status_code}") + return response.json() + + +# Usage +# game_feeds = GameFeeds() +# rec_id = game_feeds.get_available_recordings() +# session_id = game_feeds.get_session(recording_id=rec_id) +# game_feeds_data = {'rec_id': rec_id, 'session_id': session_id} +# game_boxscore = get_game_boxscore(recording_id=game_feeds_data['rec_id'], session_id=game_feeds_data['session_id']) +# game_info = get_game_info(recording_id=game_feeds_data['rec_id'], session_id=game_feeds_data['session_id']) +# game_rosters = get_game_roster(recording_id=game_feeds_data['rec_id'], session_id=game_feeds_data['session_id']) diff --git a/backend-container/src/sportsradar/simulation/session.py b/backend-container/src/sportsradar/simulation/session.py new file mode 100644 index 00000000..658af185 --- /dev/null +++ b/backend-container/src/sportsradar/simulation/session.py @@ -0,0 +1,39 @@ +import requests +from src.sportsradar.simulation.config import Config + + +def create_session(url: str, recording_id: str): + """ + Creates a session for the given recording ID. + + :param url: The URL to send the POST request to. + :param recording_id: The ID of the recording. + :return: The response from the API, or None if the request failed. + """ + headers = { + "Content-Type": f"application/{Config.CONTENT_TYPE}", + } + + json_data = { + "query": "mutation CreateSession($input: CreateSessionInput!) {\n createSession(input: $input)\n }", + "variables": { + "input": { + "recordingId": recording_id, + }, + }, + } + + try: + response = requests.post(url, headers=headers, json=json_data) + response.raise_for_status() # Will only proceed if the request was successful + except requests.RequestException as e: + print(f"Request failed with {e}") + return None + + return response + + +# Usage +# url = 'https://playback.sportradar.com/graphql' +# recording_id = '50d7e8f3-a1ce-4fcf-bb15-f8a2ad919e34' +# response = create_session(url, recording_id) diff --git a/backend-container/src/sportsradar/transform/__init__.py b/backend-container/src/sportsradar/transform/__init__.py new file mode 100644 index 00000000..8ef0fa61 --- /dev/null +++ b/backend-container/src/sportsradar/transform/__init__.py @@ -0,0 +1 @@ +"""Data Transformation using src.sportsradar.transform subpackage.""" diff --git a/backend-container/src/sportsradar/transform/additionalfeeds.py b/backend-container/src/sportsradar/transform/additionalfeeds.py new file mode 100644 index 00000000..52ad306f --- /dev/null +++ b/backend-container/src/sportsradar/transform/additionalfeeds.py @@ -0,0 +1,67 @@ +from src.sportsradar import logging_helpers + +logger = logging_helpers.get_logger(__name__) + + +class AdditionalFeedsTransformer: + """ + AdditionalFeedsTransformer class is used to transform additional feeds data. + + Args: + data (dict): A dictionary containing additional feeds data. + + Methods: + transform_weekly_depth_charts: Transforms the weekly depth charts data by removing unwanted feeds. + transform_daily_change_log: Transforms the daily change log data by removing unwanted feeds. + transform_daily_transactions: Transforms the daily transactions data by removing unwanted feeds + transform_league_hierarchy: Transforms the league hierarchy data by removing unwanted feeds + transform_seasons: Transforms the seasons data by removing unwanted feeds + transform_weekly_injuries: Transforms the weekly_injuries data by removing unwanted feeds + + + Returns: + dict: The transformed team weekly depth charts data. + dict: The transformed daily change logs data + dict: The transformed daily transactoins data + dict: The transformed league hierarchy data + dict: The transformed seasons data + dict: The Transformed weekly injuries data + """ + + UNWANTED_KEYS = ["_comment"] + + def __init__(self, data: dict): + self.data = data + + def _remove_unwanted_feeds(self): + for key in self.UNWANTED_KEYS: + if key in self.data: + self.data.pop(key) + + def transform_weekly_depth_charts(self): + self._remove_unwanted_feeds() + return self.data + + def transform_daily_change_log(self): + self._remove_unwanted_feeds() + return self.data + + def transform_daily_transactions(self): + self._remove_unwanted_feeds() + return self.data + + def transform_league_hierarchy(self): + self._remove_unwanted_feeds() + return self.data + + def transform_postgame_standings(self): + self._remove_unwanted_feeds() + return self.data + + def transform_seasons(self): + self._remove_unwanted_feeds() + return self.data + + def transform_weekly_injuries(self): + self._remove_unwanted_feeds() + return self.data diff --git a/backend-container/src/sportsradar/transform/draftfeeds.py b/backend-container/src/sportsradar/transform/draftfeeds.py new file mode 100644 index 00000000..f9433f45 --- /dev/null +++ b/backend-container/src/sportsradar/transform/draftfeeds.py @@ -0,0 +1,55 @@ +from src.sportsradar import logging_helpers + +logger = logging_helpers.get_logger(__name__) + + +class DraftFeedsTransformer: + """ + DraftFeedsTransformer class is used to transform draft feeds data. + + Attributes: + UNWANTED_KEYS (list): List of unwanted keys to be removed from the data dictionary. + + Args: + data (dict): A dictionary containing draft feeds data. + + Methods: + transform_draft_summary(): Transforms the draft summary. + transform_prospects: Transforms the prospects' data. + transform_team_draft_summary(): Transforms the team draft summary data. + transform_top_prospects(): Transforms the top prospects' data. + transform_trades(): Transforms the trades' data. + + Returns: + dict: The transformed data into dictionary. + """ + + UNWANTED_KEYS = ["_comment"] + + def __init__(self, data: dict): + self.data = data + + def _remove_unwanted_feeds(self): + for key in self.UNWANTED_KEYS: + if key in self.data: + self.data.pop(key) + + def transform_draft_summary(self): + self._remove_unwanted_feeds() + return self.data + + def transform_prospects(self): + self._remove_unwanted_feeds() + return self.data + + def transform_team_draft_summary(self): + self._remove_unwanted_feeds() + return self.data + + def transform_top_prospects(self): + self._remove_unwanted_feeds() + return self.data + + def transform_trades(self): + self._remove_unwanted_feeds() + return self.data diff --git a/backend-container/src/sportsradar/transform/gamefeeds.py b/backend-container/src/sportsradar/transform/gamefeeds.py new file mode 100644 index 00000000..0143f19f --- /dev/null +++ b/backend-container/src/sportsradar/transform/gamefeeds.py @@ -0,0 +1,42 @@ +from src.sportsradar import logging_helpers + +logger = logging_helpers.get_logger(__name__) + + +class GameFeedsTransformer: + """ + Class to transform game feeds data. + + Attributes: + UNWANTED_KEYS (list): List of unwanted keys to be removed from the data dictionary. + + Args: + data (dict): The game feeds data dictionary. + + Methods: + transform_boxscore: Transforms the boxscore data. + transform_game_roster: Transforms the game roster data. + transform_game_statistics: Transforms the game statistics data. + """ + + UNWANTED_KEYS = ["_comment"] + + def __init__(self, data: dict): + self.data = data + + def _remove_unwanted_feeds(self): + for key in self.UNWANTED_KEYS: + if key in self.data: + self.data.pop(key) + + def transform_boxscore(self): + self._remove_unwanted_feeds() + return self.data + + def transform_game_roster(self): + self._remove_unwanted_feeds() + return self.data + + def transform_game_statistics(self): + self._remove_unwanted_feeds() + return self.data diff --git a/backend-container/src/sportsradar/transform/playerfeeds.py b/backend-container/src/sportsradar/transform/playerfeeds.py new file mode 100644 index 00000000..5381aa24 --- /dev/null +++ b/backend-container/src/sportsradar/transform/playerfeeds.py @@ -0,0 +1,32 @@ +from src.sportsradar import logging_helpers + +logger = logging_helpers.get_logger(__name__) + + +class PlayerFeedsTransformer: + """ + PlayerFeedsTransformer class is used to transform player profile data. + + Args: + data (dict): A dictionary containing player profile data. + + Methods: + transform_player_profile(): Transforms the player profile data by removing unwanted feeds. + + Returns: + dict: The transformed player profile data. + """ + + UNWANTED_KEYS = ["_comment"] + + def __init__(self, data: dict): + self.data = data + + def _remove_unwanted_feeds(self): + for key in self.UNWANTED_KEYS: + if key in self.data: + self.data.pop(key) + + def transform_player_profile(self): + self._remove_unwanted_feeds() + return self.data diff --git a/backend-container/src/sportsradar/transform/primaryfeeds.py b/backend-container/src/sportsradar/transform/primaryfeeds.py new file mode 100644 index 00000000..2560b137 --- /dev/null +++ b/backend-container/src/sportsradar/transform/primaryfeeds.py @@ -0,0 +1,45 @@ +from src.sportsradar import logging_helpers + +logger = logging_helpers.get_logger(__name__) + + +class PrimaryFeedsTransformer: + """ + Class to transform player feeds data. + + Attributes: + UNWANTED_KEYS(list): List of unwanted keys to be removed from the data dictonary. + + Args: + data (dict): The game feeds data dictionary. + + Methods: + transform_current_season_schedule: Transform the current schedule data + ransform_current_week_schedule : Transform the current week schedule data + """ + + UNWANTED_KEYS = ["_comment"] + + def __init__(self, data: dict): + self.data = data + + def _remove_unwanted_feeds(self): + for key in self.UNWANTED_KEYS: + if key in self.data: + self.data.pop(key) + + def transform_current_season_schedule(self): + self._remove_unwanted_feeds() + return self.data + + def transform_current_week_schedule(self): + self._remove_unwanted_feeds() + return self.data + + def transform_seasons_schedule(self): + self._remove_unwanted_feeds() + return self.data + + def transform_weekly_schedule(self): + self._remove_unwanted_feeds() + return self.data diff --git a/backend-container/src/sportsradar/transform/teamfeeds.py b/backend-container/src/sportsradar/transform/teamfeeds.py new file mode 100644 index 00000000..ca94df42 --- /dev/null +++ b/backend-container/src/sportsradar/transform/teamfeeds.py @@ -0,0 +1,40 @@ +from src.sportsradar import logging_helpers + +logger = logging_helpers.get_logger(__name__) + + +class TeamFeedsTransformer: + """ + TeamFeedsTransformer class is used to transform team feeds data. + + Args: + data (dict): A dictionary containing team roster data. + + Methods: + transform_team_roster(): Transforms the team roster data by removing unwanted feeds. + + Returns: + dict: The transformed team roster data. + """ + + UNWANTED_KEYS = ["_comment"] + + def __init__(self, data: dict): + self.data = data + + def _remove_unwanted_feeds(self): + for key in self.UNWANTED_KEYS: + if key in self.data: + self.data.pop(key) + + def transform_team_roster(self): + self._remove_unwanted_feeds() + return self.data + + def transform_seasonal_statistics(self): + self._remove_unwanted_feeds() + return self.data + + def transform_team_profile(self): + self._remove_unwanted_feeds() + return self.data diff --git a/backend-container/src/sportsradar/workspace/datastore.py b/backend-container/src/sportsradar/workspace/datastore.py index 2b2c630e..14035272 100644 --- a/backend-container/src/sportsradar/workspace/datastore.py +++ b/backend-container/src/sportsradar/workspace/datastore.py @@ -1,18 +1,32 @@ -from pymongo import MongoClient -from pymongo.server_api import ServerApi +import os +from typing import List, Dict from urllib3.util.retry import Retry from pydantic import HttpUrl from requests.adapters import HTTPAdapter import requests +from pymongo import MongoClient +from pymongo.server_api import ServerApi + from src.sportsradar import logging_helpers -import os logger = logging_helpers.get_logger(__name__) + # load_dotenv("../../../../../.env") def create_mongo_client(): + """Create a MongoDB client connection. + + This method retrieves the MongoDB URL from the environment variable MONGODB_URL. + It then creates a MongoClient instance using the retrieved URL, along with a specified server API version and default port. + + Returns: + MongoClient: An instance of the MongoDB client. + + Raises: + ValueError: If the MONGODB_URL environment variable is not set. + """ mongo_url = os.getenv("MONGODB_URL") if mongo_url is None: raise ValueError("MongoDB environment variable for URL not set.") @@ -20,6 +34,14 @@ def create_mongo_client(): def setup_http_session(): + """ + + This function sets up an HTTP session with retries and mounts an adapter for handling HTTP requests. + + Returns: + session (requests.Session): The initialized HTTP session. + + """ retries = Retry( total=3, backoff_factor=2, status_forcelist=[429, 500, 502, 503, 504] ) @@ -31,15 +53,45 @@ def setup_http_session(): def save_data(response, db_uri, database, collection): + """ + Save data to MongoDB collection. + + Parameters: + - response (object): The response object containing the data to be saved. + - db_uri (str): The MongoDB connection URI. + - database (str): The name of the database in which to save the data. + - collection (str): The name of the collection in which to save the data. + + Raises: + - ValueError: If the database name is None. + + """ mongo_client = MongoClient(host=db_uri, server_api=ServerApi("1"), port=27017) if database is None: raise ValueError("MongoDB environment variable not set.") - print(database) - db = mongo_client[database] - db[collection].insert_one(response.json()) + else: + print(database) + db = mongo_client[database] + db[collection].insert_one(response.json()) class SportsRadarFetcher: + """ + Class to fetch data from SportsRadar API. + + Attributes: + - timeout (float): The timeout value for HTTP requests in seconds. + - http (Session): The HTTP session object for making requests. + - mongo_client (MongoClient): The MongoDB client object for connecting to the database. + - mongo_db (str): The name of the MongoDB database. + + Methods: + - __init__(timeout: float = 30): Initializes the SportsRadarFetcher instance. + - _fetch_from_url(url: HttpUrl) -> requests.Response: Fetches data from the given URL. + - fetch_data(url: HttpUrl) -> requests.Response: Fetches data from the SportsRadar API. + + """ + def __init__(self, timeout: float = 30): self.timeout = timeout self.http = setup_http_session() @@ -48,10 +100,6 @@ def __init__(self, timeout: float = 30): if self.mongo_db is None: raise ValueError("MongoDB environment variable for Database not set.") - # def save_data(self, collection, data): - # db = self.mongo_client[self.mongo_db] - # db[collection].insert_one(data) - def _fetch_from_url(self, url: HttpUrl) -> requests.Response: logger.info(f"Retrieving {url} from SportsRadar") response = self.http.get(url, timeout=self.timeout) @@ -69,11 +117,25 @@ def fetch_data(self, url: HttpUrl) -> requests.Response: class DataStore: + """ + Class for storing and retrieving data using a specified fetcher. + + Parameters: + - fetcher (SportsRadarFetcher): The fetcher object used to retrieve data. + + Attributes: + - fetcher (SportsRadarFetcher): The fetcher object used to retrieve data. + - data (dict): A dictionary to store the fetched data. + + Methods: + - fetch_data(url): Fetches data from the specified URL using the fetcher object. Returns the response if the request is successful. + - get_data_from_database(collection): Retrieves data from the specified collection in the database. Returns a list of dictionaries. + """ + def __init__(self, fetcher: SportsRadarFetcher): self.fetcher = fetcher self.data = {} - # other methods remain same def fetch_data(self, url): try: response = self.fetcher.fetch_data(url) @@ -83,3 +145,13 @@ def fetch_data(self, url): except requests.exceptions.RequestException as e: logger.error(f"Error fetching URL {url}: {str(e)}") raise + + def get_data_from_database(self, collection: str) -> List[Dict]: + try: + client = self.fetcher.mongo_client + db = client[self.fetcher.mongo_db] + data = db[collection].find() + return list(data) + except Exception as e: + logger.error(f"Error getting data from MONGODB_DATABASE: {str(e)}") + raise diff --git a/backend-container/src/sportsradar/workspace/resource_cache.py b/backend-container/src/sportsradar/workspace/resource_cache.py new file mode 100644 index 00000000..f70f8ffe --- /dev/null +++ b/backend-container/src/sportsradar/workspace/resource_cache.py @@ -0,0 +1,229 @@ +import datetime +from abc import ABC, abstractmethod +from typing import Any, NamedTuple, Optional, cast +from dotenv import load_dotenv, find_dotenv +import redis +import json + +from src.sportsradar.logging_helpers import get_logger + +logger = get_logger(__name__) +load_dotenv(find_dotenv()) + + +class NFLStatsResourceKey(NamedTuple): + """ + Represents a resource key for NFL statistics. + + Attributes: + None + + Methods: + __str__(): Returns the string representation of the resource key. + + Usage: + resource_key = NFLStatsResourceKey() + key_str = str(resource_key) + """ + + def __str__(self) -> str: + return "NFLStats_" + datetime.date.today().strftime("%Y-%m-%d") + + +class AbstractCache(ABC): + """ + AbstractCache + + This class is an abstract base class for cache implementations. It defines the common interface that all cache implementations should adhere to. + + Attributes: + _read_only (bool): Indicates whether the cache is read-only or not. + + Methods: + __init__(self, read_only: bool = False): + Initializes the AbstractCache object. + + Args: + read_only (bool, optional): Indicates whether the cache is read-only. Defaults to False. + + is_read_only(self) -> bool: + Returns the read-only status of the cache. + + Returns: + bool: True if the cache is read-only, False otherwise. + + get(self, resource: NFLStatsResourceKey) -> Any: + Retrieves the content associated with the given resource key from the cache. + + Args: + resource (NFLStatsResourceKey): The resource key to retrieve the content for. + + Returns: + Any: The content associated with the given resource key. + + add(self, resource: NFLStatsResourceKey, content: Any) -> None: + Adds the content to the cache under the given resource key. + + Args: + resource (NFLStatsResourceKey): The resource key to add the content under. + content (Any): The content to be added to the cache. + + Returns: + None + + delete(self, resource: NFLStatsResourceKey) -> None: + Deletes the content associated with the given resource key from the cache. + + Args: + resource (NFLStatsResourceKey): The resource key to delete the content for. + + Returns: + None + + contains(self, resource: NFLStatsResourceKey) -> bool: + Checks whether the cache contains the content associated with the given resource key. + + Args: + resource (NFLStatsResourceKey): The resource key to check for. + + Returns: + bool: True if the cache contains the content for the given resource key, False otherwise. + """ + + def __init__(self, read_only: bool = False): + self._read_only = read_only + + def is_read_only(self) -> bool: + return self._read_only + + @abstractmethod + def get(self, resource: NFLStatsResourceKey) -> Any: + pass + + @abstractmethod + def add(self, resource: NFLStatsResourceKey, content: Any) -> None: + pass + + @abstractmethod + def delete(self, resource: NFLStatsResourceKey) -> None: + pass + + @abstractmethod + def contains(self, resource: NFLStatsResourceKey) -> bool: + pass + + +class RedisCache(AbstractCache): + """ + RedisCache + + A class that represents a cache using Redis as the underlying storage. + + Methods: + - get(resource: NFLStatsResourceKey) -> Any: + Retrieves the value associated with the given resource key from the cache. + If the key does not exist in the cache, a KeyError is raised. + + - add(resource: NFLStatsResourceKey, value: Any): + Adds the given resource key-value pair to the cache. + If the cache is read-only, the operation is ignored. + + - delete(resource: NFLStatsResourceKey): + Deletes the value associated with the given resource key from the cache. + If the cache is read-only, the operation is ignored. + + - contains(resource: NFLStatsResourceKey) -> bool: + Checks whether the cache contains the given resource key. + + Parameters: + - host: Optional[str] = None + The host address of the Redis server. If not provided, the default value is None. + + - port: Optional[str] = None + The port number of the Redis server. If not provided, the default value is None. + + - password: Optional[str] = None + The password to authenticate with the Redis server. If not provided, the default value is None. + + - **kwargs: Any + Additional keyword arguments to customize the cache. + + Attributes: + - _db: redis.StrictRedis + The Redis client used for interacting with the Redis server. + + Note: + This class is designed to be used as a subclass of AbstractCache. + It assumes the existence of the Redis package in the Python environment. + The Redis server must be accessible for the cache operations to work. + + Example usage: + cache = RedisCache(host='localhost', port='6379', password='password') + cache.add('resource_key', {'data': 'value'}) + value = cache.get('resource_key') + cache.contains('resource_key') + + Caution: + Be cautious when using this cache implementation with sensitive data as Redis is an in-memory database. + Ensure proper security measures are in place to protect the data stored in Redis. + """ + + def __init__( + self, + host: Optional[str] = None, + port: Optional[str] = None, + password: Optional[str] = None, + **kwargs: Any, + ): + super().__init__(**kwargs) + self._db = redis.StrictRedis( + host=cast(str, host), port=cast(str, port), password=cast(str, password) + ) + + def get(self, resource: NFLStatsResourceKey) -> Any: + value = self._db.get(str(resource)) + if value is None: + raise KeyError(f"Resource - {resource} not found in the Redis cache") + return json.loads(value) + + def add(self, resource: NFLStatsResourceKey, value: Any): + if self.is_read_only(): + logger.debug(f"Read-only cache: ignoring set({resource})") + return + self._db.set(str(resource), json.dumps(value)) + + def delete(self, resource: NFLStatsResourceKey): + if self.is_read_only(): + logger.debug(f"Read-only cache: ignoring delete({resource})") + return + self._db.delete(str(resource)) + + def contains(self, resource: NFLStatsResourceKey) -> bool: + return bool(self._db.exists(str(resource))) + + +# example usage +# document_to_store = { +# "name": "John", +# "documents": [ +# {"title": "Doc 1", "content": "Content 1"}, +# {"title": "Doc 2", "content": "Content 2"}, +# ], +# } +# +# redis_host = os.getenv("REDIS_HOST_GG") +# redis_port = os.getenv("REDIS_PORT_GG") +# redis_pass = os.getenv("REDIS_PASS_GG") +# +# nfl_stats_resource_key = NFLStatsResourceKey() +# +# redis_cache = RedisCache(host=redis_host, port=redis_port, password=redis_pass) +# +# redis_cache.add(resource=nfl_stats_resource_key, value=document_to_store) +# +# try: +# retrieved_content = redis_cache.get(resource=nfl_stats_resource_key) +# print(retrieved_content) +# +# except KeyError: +# print("Resource not found in the Redis cache") diff --git a/frontend-container/package-lock.json b/frontend-container/package-lock.json index 8518709e..6245d46d 100644 --- a/frontend-container/package-lock.json +++ b/frontend-container/package-lock.json @@ -31,7 +31,7 @@ "@reduxjs/toolkit": "2.0.1", "@rollup/plugin-json": "^6.1.0", "abort-controller": "^3.0.0", - "amazon-cognito-identity-js": "6.3.7", + "amazon-cognito-identity-js": "^6.3.8", "apexcharts": "3.45.0", "apollo-server-express": "^3.13.0", "atlas-app-services-cli": "^1.0.3", @@ -6700,9 +6700,9 @@ } }, "node_modules/amazon-cognito-identity-js": { - "version": "6.3.7", - "resolved": "https://registry.npmjs.org/amazon-cognito-identity-js/-/amazon-cognito-identity-js-6.3.7.tgz", - "integrity": "sha512-tSjnM7KyAeOZ7UMah+oOZ6cW4Gf64FFcc7BE2l7MTcp7ekAPrXaCbpcW2xEpH1EiDS4cPcAouHzmCuc2tr72vQ==", + "version": "6.3.8", + "resolved": "https://registry.npmjs.org/amazon-cognito-identity-js/-/amazon-cognito-identity-js-6.3.8.tgz", + "integrity": "sha512-dM8/9YhXYn/X3y+MejKA3XXTfysNKeW++D2PeNpoT+83M+DGxGbVsn60FPSv8Pbk5Clz8jw7RDchlIRufye1Mg==", "dependencies": { "@aws-crypto/sha256-js": "1.2.2", "buffer": "4.9.2", diff --git a/frontend-container/package.json b/frontend-container/package.json index 38a66294..546a4171 100644 --- a/frontend-container/package.json +++ b/frontend-container/package.json @@ -35,7 +35,7 @@ "@reduxjs/toolkit": "2.0.1", "@rollup/plugin-json": "^6.1.0", "abort-controller": "^3.0.0", - "amazon-cognito-identity-js": "6.3.7", + "amazon-cognito-identity-js": "6.3.8", "apexcharts": "3.45.0", "apollo-server-express": "^3.13.0", "atlas-app-services-cli": "^1.0.3", diff --git a/frontend-container/src/assets/img/logo.png b/frontend-container/src/assets/img/logo.png index 44333621..c912be70 100644 Binary files a/frontend-container/src/assets/img/logo.png and b/frontend-container/src/assets/img/logo.png differ