diff --git a/current/.gitignore b/current/.gitignore new file mode 100644 index 0000000..8e408ff --- /dev/null +++ b/current/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +*.pyc +*.pyo +data/ +*.pkl +*.pkl.bak +.env diff --git a/current/backend/OPR.py b/current/backend/OPR.py new file mode 100644 index 0000000..32d658d --- /dev/null +++ b/current/backend/OPR.py @@ -0,0 +1,89 @@ +from scipy.sparse import csr_array +import numpy as np +from scipy.linalg import lstsq +#from scipy.sparse.linalg import spsolve, norm +#from scipy.sparse.linalg import lsqr + + +class OPR: + @staticmethod + def get_rows(m): + yield m.alliances.red.team_keys, m.alliances.blue.team_keys, m.score_breakdown['red']['totalPoints'], m.score_breakdown['blue']['totalPoints'] + + def __init__(self, matches, teams: set): + team_lookup = dict((k,i) for (i,k) in enumerate(teams)) + + A_data = [] + row = [] + col = [] + b_OPR = [] + b_DPR = [] + b_TPR = [] + ctr = 0 + for m in map(OPR.get_rows, matches): + for red_keys, blue_keys, red_score, blue_score in (m): + for t in red_keys: + row.append(len(b_OPR)) + col.append(team_lookup[t]) + A_data.append(1) + b_OPR.append(red_score) + b_DPR.append(blue_score) + b_TPR.append(red_score-blue_score) + for t in blue_keys: + row.append(len(b_OPR)) + col.append(team_lookup[t]) + A_data.append(1) + b_OPR.append(blue_score) + b_DPR.append(red_score) + b_TPR.append(blue_score-red_score) + + + A = csr_array((A_data, (row, col)), shape=(len(b_OPR), len(team_lookup))) + # print(A.shape, b.shape ) + #x = spsolve(A, b) + #x + + # Thanks ChatGPT! + result = {} + for (b, tag) in [(b_OPR, 'OPR'), (b_DPR, 'DPR'), (b_TPR, 'TPR')]: + b = np.array(b) + x, residuals, rank, s = lstsq(A.todense(), b) + RSS = residuals.sum() + Rinv = np.linalg.inv(np.triu(s)) + err = np.mean(A@x-b) + print(f'Error {tag}: {err}') + + sigmas = np.sqrt(RSS / (len(b) - len(x)) * np.diag(Rinv)) + result[tag] = (x, sigmas) + #return_values = lsqr(A, b, calc_var=True) + #result = return_values + #print(result) + #x = return_values[0] + #var = return_values[-1] + #print(var) + + #for t,opr,sigma in sorted(opr, key=lambda x: x[1], reverse=True): + # print(t,opr,sigma) + + #print((A@x).shape,b.shape) + #print(A@x-b) + #self.opr = [(t,x[i],sigmas[i]) for i,t in enumerate(teams)] + self.opr_lookup = dict ([(t, {'ix':i}) for i,t in enumerate(teams)]) + for t in self.opr_lookup: + self.opr_lookup[t]['opr'] = {'mu': result['OPR'][0][self.opr_lookup[t]['ix']], 'sigma': result['OPR'][1][self.opr_lookup[t]['ix']]} + self.opr_lookup[t]['dpr'] = {'mu': result['DPR'][0][self.opr_lookup[t]['ix']], 'sigma': result['DPR'][1][self.opr_lookup[t]['ix']]} + self.opr_lookup[t]['tpr'] = {'mu': result['TPR'][0][self.opr_lookup[t]['ix']], 'sigma': result['TPR'][1][self.opr_lookup[t]['ix']]} + self.opr_lookup[''] = {'opr': {'mu': 0, 'sigma': 0}, 'dpr': {'mu': 0, 'sigma': 0}, 'tpr': {'mu': 0, 'sigma': 0}} + + def predict(self, red,blue, method='opr'): + mu = [] + sigma = [] + for r in red: + mu.append(self.opr_lookup[r][method]['mu']) + sigma.append(self.opr_lookup[r][method]['sigma']) + for b in blue: + mu.append(-self.opr_lookup[b][method]['mu']) + sigma.append(self.opr_lookup[b][method]['sigma']) + mu = sum(mu) + sigma = np.linalg.norm(sigma) + return(mu,sigma) \ No newline at end of file diff --git a/current/backend/TBA.py b/current/backend/TBA.py new file mode 100644 index 0000000..ee7eca4 --- /dev/null +++ b/current/backend/TBA.py @@ -0,0 +1,190 @@ +from __future__ import print_function +import pickle +import os +import logging +import swagger_client as v3client +from swagger_client.rest import ApiException + +logging.basicConfig(level=logging.INFO) + +class TBA: + def __init__(self, year=2026, district='all'): + self.matches = None + self.year = year + self.district = district + self.DATA_FOLDER = os.environ.get('DATA_FOLDER', './data') + if not os.path.exists(self.DATA_FOLDER): + os.makedirs(self.DATA_FOLDER) + self.matches_file = f'{self.DATA_FOLDER}/matches_{self.district}_{self.year}.pkl' + + # Configure API key authorization: apiKey + configuration = v3client.Configuration() + configuration.api_key['X-TBA-Auth-Key'] = os.environ.get('TBA_API_KEY') + self.configuration = configuration + + self.api_instance = v3client.EventApi(v3client.ApiClient(configuration)) + + if not os.path.exists(self.matches_file): + self.fetch_all_matches() + + with open(self.matches_file, 'rb') as f: + self.matches = pickle.load(f) + self.matches['last_modified'] = os.stat(self.matches_file).st_mtime + + def fetch_all_matches(self, eventsToPull="", reset=False): + """ + Fetch all matches for the configured year, filtered to eventsToPull (or all + events if empty). Only events whose data has changed since the last fetch are + re-downloaded; everything else is served from the local cache. + + Incremental behaviour is implemented via per-event HTTP If-Modified-Since + headers. The Last-Modified value returned by TBA for each event is stored + in result['event_last_modified'][event_key] and reused on the next call so + that TBA can return 304 Not Modified for unchanged events. + + Set reset=True to ignore all cached timestamps and force a full re-fetch. + """ + api_instance = self.api_instance + result = {} + events_filter = None + if eventsToPull != "": + events_filter = eventsToPull.split(',') + + outfile = self.matches_file + + # Load the existing cache so we can do incremental updates. + if os.path.exists(outfile): + with open(outfile, 'rb') as inresult: + try: + result = pickle.load(inresult) + except Exception as e: + logging.error('Failed to load prior matches: %s', e) + result = {} + + # Ensure the sub-dicts we rely on always exist in result. + result.setdefault('matches', {}) + result.setdefault('event_teams', {}) + result.setdefault('event_last_modified', {}) + + # --- Fetch the list of events --- + # Use the globally stored Last-Modified for the events-list endpoint. + events_if_modified = '' + if not reset and 'headers' in result and 'Last-Modified' in result['headers']: + events_if_modified = result['headers']['Last-Modified'] + + events = result.get('events', []) + try: + fetched_events = api_instance.get_events_by_year( + self.year, if_modified_since=events_if_modified) + if self.district != 'all': + fetched_events = [ + e for e in fetched_events + if e.district and e.district.abbreviation == self.district + ] + if events_filter is not None: + fetched_events = [e for e in fetched_events if e.key in events_filter] + + # Update the events list and the global Last-Modified header. + events = fetched_events + result['events'] = events + result['headers'] = api_instance.api_client.last_response.getheaders() + logging.info('Fetched %d events for %d', len(events), self.year) + except ApiException as exc: + if exc.status == 304: + logging.info('Events list not modified since last fetch; using cache') + else: + logging.error('Error fetching events for year %d: %s', self.year, exc) + + # --- Fetch matches and teams for each event incrementally --- + for event in events: + # Re-use the per-event Last-Modified so TBA can skip unchanged events. + event_if_modified = '' if reset else result['event_last_modified'].get(event.key, '') + + # Matches + try: + matches = api_instance.get_event_matches( + event.key, if_modified_since=event_if_modified) + result['matches'][event.key] = matches + last_mod = api_instance.api_client.last_response.getheader('Last-Modified', '') + if last_mod: + result['event_last_modified'][event.key] = last_mod + logging.info('Fetched %d matches for event %s', len(matches), event.key) + except ApiException as exc: + if exc.status == 304: + logging.info('Matches for event %s not modified; using cache', event.key) + else: + logging.error('Error fetching matches for event %s: %s', event.key, exc) + + # Teams + try: + teams = api_instance.get_event_teams( + event.key, if_modified_since=event_if_modified) + result['event_teams'][event.key] = teams + except ApiException as exc: + if exc.status == 304: + logging.info('Teams for event %s not modified; using cache', event.key) + else: + logging.error('Error fetching teams for event %s: %s', event.key, exc) + + # Persist the updated cache. + if 'events' in result: + if os.path.exists(outfile): + os.replace(outfile, outfile + '.bak') + with open(outfile, 'wb') as outmatches: + pickle.dump(result, outmatches) + + result['last_modified'] = os.stat(outfile).st_mtime + self.matches = result + return result + + def fetch_events(self, team_key='frc492', if_modified_since=''): + events = self.api_instance.get_team_events_by_year( + team_key, self.year, if_modified_since=if_modified_since) + for e in events: + print(f'{e.event_code}\t{e.name}\t{e.start_date}') + return events + + def fetch_event_rankings(self, event_key): + rankings = self.api_instance.get_event_rankings(event_key) + return rankings + + def fetch_event_teams(self, event_key): + return self.api_instance.get_event_teams(event_key) + + def fetch_matches(self, team_key='frc492', if_modified_since=''): + """ + Fetches all matches for all events associated with a single team. + """ + result = [] + try: + events = self.api_instance.get_team_events_by_year( + team_key, self.year, if_modified_since=if_modified_since) + for e in events: + print('Fetching: ' + e.short_name) + matches = self.api_instance.get_event_matches(e.key) + result += matches + except ApiException as e: + print("Exception when calling EventApi->get_team_events: %s\n" % e) + return result + + @staticmethod + def count_matches(events): + return sum([len(events[e]) for e in events]) + + def fetch_teams(self): + """ + Fetch the list of all teams for the configured year. + """ + list_api = v3client.ListApi(v3client.ApiClient(self.configuration)) + pg = 0 + result = [] + while True: + logging.info('Fetching teams page %d', pg) + teams = list_api.get_teams_by_year(self.year, pg) + if len(teams) == 0: + break + result += teams + pg += 1 + + with open(f'{self.DATA_FOLDER}/teams_{self.year}.pkl', 'wb') as outTeams: + pickle.dump(result, outTeams) diff --git a/current/backend/__init__.py b/current/backend/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/current/backend/app.py b/current/backend/app.py new file mode 100644 index 0000000..491e3d0 --- /dev/null +++ b/current/backend/app.py @@ -0,0 +1,315 @@ +from collections import Counter +import logging +import pickle +import os +from time import strftime, gmtime +from dotenv import load_dotenv +import numpy as np +from tqdm import tqdm +from flask_cors import CORS +import scipy.stats as stats +from OPR import OPR +from TBA import TBA +from flask import Flask, jsonify, request, render_template, send_from_directory + + +# TODO: +# Periodic match refresh +# Other model types +# Overall OPR rankings page +# Fix the best 2 of 3 bracket? +# button to auto-populate alliances - basic, greedy strategies +# dropdown for event selection +# enable arbitrary district selection, all district selection + +# model/match data somewhere that doesn't trigger reload. +load_dotenv() +logging.basicConfig(level=logging.DEBUG) + +DATA_FOLDER = os.getenv('DATA_FOLDER', '../') +logging.info('Using data folder: %s', DATA_FOLDER) +if not os.path.exists(DATA_FOLDER): + os.makedirs(DATA_FOLDER) + +models = {} +tba = TBA(year=2026, district='pnw') +all_matches = tba.matches + +app = Flask(__name__, static_folder='static/build') +CORS(app) + +@app.route('/', defaults={'path': ''}) +@app.route('/') +def serve(path): + logging.info('Serving %s', path) + if path != "" and os.path.exists(app.static_folder + '/' + path): + return send_from_directory(app.static_folder, path) + else: + return send_from_directory(app.static_folder, 'index.html') + +def create_model(district, event, match_type, force_recompute=False): + ''' + district: string, the district to filter by. eg 'pnw', 'all' for all districts + event: string, the event to filter by. eg. '2024wasno', 'all' for all events + match_type: string, the match type to filter by. eg. 'qm', 'all' for all match types + force_recompute: bool, if True, recompute the model even if it already exists + ''' + + model_key=f'{district}_{event}_{match_type}' + model_fn = f'{DATA_FOLDER}/model_{model_key}.pkl' + + + #if all_matches is None: + # filename = f'{DATA_FOLDER}/matches_2024.pkl' + # with open(filename, 'rb') as f: + # all_matches = pickle.load(f) + # # stat the matches file to get its last modified time and set it on all_matches + # all_matches['last_modified'] = os.stat(filename).st_mtime + + if not os.path.exists(model_fn) or force_recompute or all_matches['last_modified'] > os.stat(model_fn).st_mtime: + selected_district = [m.key for m in all_matches['events']] if district == 'all' else \ + [m.key for m in all_matches['events'] if m.district and m.district.abbreviation==district] + + print(f'{len(all_matches["matches"])} events') + + data = [m for k in all_matches['matches'] for m in all_matches['matches'][k]] + data = [m for m in data if m.winning_alliance!='' and m.score_breakdown is not None] + print(f'Found {len(data)} matches') + + def in_scope(m): + return ((event == 'all' and m.event_key in selected_district) \ + or (m.event_key == event)) \ + and (match_type == 'all' or m.comp_level == match_type) + + selected_matches = list(filter(in_scope, data)) + + teams = set() + for m in selected_matches: + for t in m.alliances.red.team_keys: + teams.add(t) + for t in m.alliances.blue.team_keys: + teams.add(t) + + teams = list(sorted(teams)) + logging.debug('Teams: %s', len(teams)) + + opr = OPR(selected_matches, teams) + opr.data_timestamp = all_matches['last_modified'] + logging.debug('Saving %s', model_fn) + with open(model_fn, 'wb') as f: + pickle.dump(opr, f) + + with open(model_fn, 'rb') as f: + logging.debug('Loading %s', model_fn) + models[model_key] = pickle.load(f) + +def get_model(model_key): + if model_key not in models: + create_model(*model_key.split('_')) + assert model_key in models + return models[model_key] + +@app.route('/model/') +def get_model_info(model_key): + ''' + model_key: string, the key for the model to get info for + returns: a json object with the model info + ''' + opr = get_model(model_key) + (district, events, match_type) = model_key.split('_') + timestamp = strftime('%Y-%m-%d %H:%M:%S', gmtime(opr.data_timestamp)) + return jsonify({ + 'district': district, 'event': events, 'match_type': match_type, 'teams': len(opr.opr_lookup), 'last_modified': timestamp}) + +@app.route('/model//refresh') +def refresh_model(model_key): + ''' + model_key: string, the key for the model to refresh + returns: a json object with the model info + ''' + global all_matches + global models + # TODO this must run async: https://stackoverflow.com/questions/14384739/how-can-i-add-a-background-thread-to-flask + tba.fetch_all_matches() + all_matches = tba.matches + models = {} + return jsonify({'all_matches': len(all_matches['matches'])}) + +# pass in the team id and get back the stats for that team. +@app.route('/model//team/') +def get_opr(model_key, team_id): + ''' + model_key: string, the key for the model to use + team_id: string, the team id to get the stats for + ''' + opr=get_model(model_key) + return jsonify({f"{team_id}": opr.opr_lookup[team_id]}) + +@app.route('/model//predict///') +def get_prediction(model_key, red,blue, method): + ''' + model_key: string, the key for the model to use + red: string, a comma separated list of red teams + blue: string, a comma separated list of blue teams + method: opr, dpr, or tpr + returns: a json object with the predicted spread + and standard deviation in favor of the red alliance + ''' + red = red.split(',') + blue = blue.split(',') + opr = get_model(model_key) + (spread, sigma) = opr.predict(red,blue, method=method) + pRed = 1.0-stats.norm.cdf(0, loc=spread, scale=sigma) + return jsonify({'red': red, 'blue': blue, 'spread':spread, 'sigma':sigma, 'pRed':pRed}) + +@app.route('/model//teams') +def get_teams(model_key): + ''' + model_key: string, the key for the model to use + returns: a json object with the list of teams in the model + ''' + opr = get_model(model_key) + return jsonify({'teams': list(opr.opr_lookup.keys())}) + +@app.route('/model//event//teams') +def get_event_teams(model_key, event_key): + ''' + model_key: string, the key for the model to use + event_key: string, the key for the event to get the teams for + returns: a json object with the list of teams in the event + ''' + logging.info('Getting teams for model %s event %s', model_key, event_key) + opr = get_model(model_key) + + teams = tba.fetch_event_teams(event_key) + + EMPTY_OPR = {'opr': {'mu': 0, 'sigma': 0}, 'dpr': {'mu': 0, 'sigma': 0}, 'tpr': {'mu': 0, 'sigma': 0}} + + result = [ + { + 'team': t.key, + 'nickname': t.nickname, + 'number': t.team_number, + 'stats': opr.opr_lookup[t.key] if t.key in opr.opr_lookup else EMPTY_OPR + } + for t in teams + ] + + return jsonify(result) + + +@app.route('/model//bracket/', methods=['POST']) +def run_bracket(model_key, model_method): + ''' + POST method to run a playoff bracket + model_key: string, the key for the model to use + model_method: one of opr, tpr, dpr + post body: json object containing the set of alliances, of the format: + {'A1': [team1, team2, team3], 'A2': [...], ..., 'A8': [...]} + returns: a json object with the predicted spread + and standard deviation for each match in the bracket + ''' + alliances = request.get_json() + opr = get_model(model_key) + logging.debug('Running bracket for model %s', model_key) + # sanity check the alliances for presence in the model + for a in alliances: + for t in alliances[a]: + if t not in opr.opr_lookup: + # send a bad request result (400) if a team is not found + logging.error('Team %s not found in model %s', t, model_key) + return jsonify({'error': f'Team {t} not found in model {model_key}'}) + reverse_lookup = {str(v):k for k,v in alliances.items()} + + bracket = { + 1: ['A1', 'A8'], + 2: ['A4', 'A5'], + 3: ['A2', 'A7'], + 4: ['A3', 'A6'], + 5: ['L1', 'L2'], + 6: ['L3', 'L4'], + 7: ['W1', 'W2'], + 8: ['W3', 'W4'], + 9: ['L7', 'W6'], + 10: ['W5', 'L8'], + 11: ['W10', 'W9'], + 12: ['W7', 'W8'], + 13: ['L12', 'W11'], + 14: ['W12', 'W13'], + 15: ['W14', 'L14'], + 16: ['W15', 'L15'] + } + + density = {i:Counter() for i in range(1,len(bracket)+1)} + + def runMatch(matchNumber): + red_id,blue_id = bracket[matchNumber] + + red = alliances[red_id] + blue =alliances[blue_id] + density[matchNumber][reverse_lookup[str(red)]]+=1 + density[matchNumber][reverse_lookup[str(blue)]]+=1 + #density[matchNumber][red_id]+=1 + #density[matchNumber][blue_id]+=1 + + # mu and sigma are the expected advantage for red + mu,sigma = opr.predict(red,blue, method=model_method) + r = np.random.normal(mu, sigma) + + if r>0: + winner = red + loser = blue + else: + winner = blue + loser = red + alliances[f'W{matchNumber}'] = winner + alliances[f'L{matchNumber}'] = loser + + + def pMatch(matchNumber): + red_id,blue_id = bracket[matchNumber] + + red = alliances[red_id] + blue =alliances[blue_id] + density[matchNumber][reverse_lookup[str(red)]]+=1 + density[matchNumber][reverse_lookup[str(blue)]]+=1 + #density[matchNumber][red_id]+=1 + #density[matchNumber][blue_id]+=1 + + # mu and sigma are the expected advantage for red + return opr.predict(red,blue) + + + + def pRed(matchNumber): + mu,sigma = pMatch(matchNumber) + return 1.0-stats.norm.cdf(0, loc=mu, scale=sigma) + + + def runBracket(): + for i in range(1,17): + runMatch(i) + wins = Counter() + for i in range(14,17): + w = alliances[f'W{i}'] + wins[str(w)]+=1 + return sorted(wins, reverse=True, key=lambda x: wins[x])[0], (alliances['A6'] in [alliances['W11'],alliances['W13']]) + + overall = Counter() + inFinalCtr = 0 + for b in tqdm(range(1000)): + (w, inFinal) = runBracket() + overall[reverse_lookup[str(w)]] += 1 + inFinalCtr += 1 if inFinal else 0 + + for k in sorted(overall, key=lambda x: overall[x], reverse=True): + print(k, overall[k]) + + print(f'inFinal: {inFinalCtr}') + + for k in sorted(density): + print(k, density[k]) + return jsonify({'overall':overall, 'density':density}) + +if __name__ == '__main__': + app.run(debug=False) \ No newline at end of file diff --git a/current/backend/requirements.txt b/current/backend/requirements.txt new file mode 100644 index 0000000..c190a4a --- /dev/null +++ b/current/backend/requirements.txt @@ -0,0 +1,5 @@ +python-dotenv +flask +flask-cors +scipy +numpy \ No newline at end of file diff --git a/current/backend/swagger_client b/current/backend/swagger_client new file mode 120000 index 0000000..0ac7f3f --- /dev/null +++ b/current/backend/swagger_client @@ -0,0 +1 @@ +../../2024/backend/swagger_client \ No newline at end of file diff --git a/current/backend/test_TBA.py b/current/backend/test_TBA.py new file mode 100644 index 0000000..f48f050 --- /dev/null +++ b/current/backend/test_TBA.py @@ -0,0 +1,243 @@ +""" +Tests for the incremental TBA match-caching logic in current/backend/TBA.py. + +These tests use unittest.mock to avoid real HTTP calls and verify that: + 1. Per-event Last-Modified headers are stored and reused. + 2. A 304 Not Modified response for an event preserves cached data. + 3. The events-list 304 falls back to the cached events list. + 4. reset=True ignores all cached timestamps. + 5. The merge logic never loses previously cached events. +""" +import os +import sys +import unittest +from unittest.mock import MagicMock, patch + +# Make sure the current/backend package is importable. +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + +from swagger_client.rest import ApiException +from TBA import TBA + + +def _make_api_exception(status): + return ApiException(status=status, reason='test') + + +def _make_event(key, district_abbrev=None): + event = MagicMock() + event.key = key + if district_abbrev: + event.district = MagicMock() + event.district.abbreviation = district_abbrev + else: + event.district = None + return event + + +class _BaseTBATest(unittest.TestCase): + """ + Base class that patches v3client, os.path.exists, pickle.load/dump and + os.replace so that no real file I/O or network calls are made. + """ + + def setUp(self): + # --- API client patches --- + self.config_patcher = patch('TBA.v3client.Configuration') + self.event_api_patcher = patch('TBA.v3client.EventApi') + self.api_client_patcher = patch('TBA.v3client.ApiClient') + self.mock_config_cls = self.config_patcher.start() + self.mock_event_api_cls = self.event_api_patcher.start() + self.mock_api_client_cls = self.api_client_patcher.start() + + self.mock_api = MagicMock() + self.mock_event_api_cls.return_value = self.mock_api + self.fake_last_response = MagicMock() + self.fake_last_response.getheaders.return_value = { + 'Last-Modified': 'Thu, 01 Jan 2026 00:00:00 GMT' + } + self.fake_last_response.getheader.return_value = 'Thu, 01 Jan 2026 00:00:00 GMT' + self.mock_api.api_client.last_response = self.fake_last_response + + # --- File I/O patches --- + # Start with an existing, empty cache so __init__ doesn't trigger a fetch. + self._initial_cache = { + 'events': [], 'matches': {}, 'event_teams': {}, + 'event_last_modified': {} + } + self.exists_patcher = patch('TBA.os.path.exists', return_value=True) + self.stat_patcher = patch('TBA.os.stat') + self.replace_patcher = patch('TBA.os.replace') + self.makedirs_patcher = patch('TBA.os.makedirs') + self.open_patcher = patch('builtins.open', unittest.mock.mock_open()) + self.pickle_load_patcher = patch('TBA.pickle.load', + side_effect=self._pickle_load_side_effect) + self.pickle_dump_patcher = patch('TBA.pickle.dump') + + self.mock_exists = self.exists_patcher.start() + self.mock_stat = self.stat_patcher.start() + self.mock_replace = self.replace_patcher.start() + self.mock_makedirs = self.makedirs_patcher.start() + self.mock_open = self.open_patcher.start() + self.mock_pickle_load = self.pickle_load_patcher.start() + self.mock_pickle_dump = self.pickle_dump_patcher.start() + + mock_stat_result = MagicMock() + mock_stat_result.st_mtime = 1234567890.0 + self.mock_stat.return_value = mock_stat_result + + with patch.dict(os.environ, {'DATA_FOLDER': '/tmp/tba_test', 'TBA_API_KEY': 'test'}): + self.tba = TBA(year=2026, district='all') + + def _pickle_load_side_effect(self, f): + return dict(self._initial_cache) # return a fresh copy each time + + def tearDown(self): + self.config_patcher.stop() + self.event_api_patcher.stop() + self.api_client_patcher.stop() + self.exists_patcher.stop() + self.stat_patcher.stop() + self.replace_patcher.stop() + self.makedirs_patcher.stop() + self.open_patcher.stop() + self.pickle_load_patcher.stop() + self.pickle_dump_patcher.stop() + + +class TestIncrementalFetch(_BaseTBATest): + # ------------------------------------------------------------------ + # Test 1: per-event Last-Modified is stored after a fresh fetch + # ------------------------------------------------------------------ + def test_per_event_last_modified_stored(self): + event_a = _make_event('2026waaaa') + self.mock_api.get_events_by_year.return_value = [event_a] + self.mock_api.get_event_matches.return_value = [] + self.mock_api.get_event_teams.return_value = [] + self.fake_last_response.getheader.return_value = 'Fri, 02 Jan 2026 12:00:00 GMT' + + result = self.tba.fetch_all_matches() + + self.assertIn('2026waaaa', result['event_last_modified']) + self.assertEqual(result['event_last_modified']['2026waaaa'], + 'Fri, 02 Jan 2026 12:00:00 GMT') + + # ------------------------------------------------------------------ + # Test 2: 304 for an event's matches preserves the cached matches + # ------------------------------------------------------------------ + def test_event_304_preserves_cached_matches(self): + event_a = _make_event('2026waaaa') + cached_match = MagicMock() + self._initial_cache = { + 'events': [event_a], + 'matches': {'2026waaaa': [cached_match]}, + 'event_teams': {'2026waaaa': []}, + 'event_last_modified': {'2026waaaa': 'Mon, 01 Jan 2026 00:00:00 GMT'}, + 'headers': {'Last-Modified': 'Mon, 01 Jan 2026 00:00:00 GMT'}, + } + + self.mock_api.get_events_by_year.return_value = [event_a] + self.mock_api.get_event_matches.side_effect = _make_api_exception(304) + self.mock_api.get_event_teams.side_effect = _make_api_exception(304) + + result = self.tba.fetch_all_matches() + + self.assertIn('2026waaaa', result['matches']) + self.assertEqual(result['matches']['2026waaaa'], [cached_match]) + + # ------------------------------------------------------------------ + # Test 3: events-list 304 falls back to cached events + # ------------------------------------------------------------------ + def test_events_list_304_uses_cached_events(self): + event_a = _make_event('2026waaaa') + self._initial_cache = { + 'events': [event_a], + 'matches': {}, + 'event_teams': {}, + 'event_last_modified': {}, + 'headers': {'Last-Modified': 'Mon, 01 Jan 2026 00:00:00 GMT'}, + } + + self.mock_api.get_events_by_year.side_effect = _make_api_exception(304) + self.mock_api.get_event_matches.return_value = [] + self.mock_api.get_event_teams.return_value = [] + + result = self.tba.fetch_all_matches() + + self.assertIn(event_a, result['events']) + + # ------------------------------------------------------------------ + # Test 4: reset=True sends empty if_modified_since for every event + # ------------------------------------------------------------------ + def test_reset_ignores_cached_timestamps(self): + event_a = _make_event('2026waaaa') + self._initial_cache = { + 'events': [event_a], + 'matches': {'2026waaaa': []}, + 'event_teams': {}, + 'event_last_modified': {'2026waaaa': 'Mon, 01 Jan 2026 00:00:00 GMT'}, + 'headers': {'Last-Modified': 'Mon, 01 Jan 2026 00:00:00 GMT'}, + } + + self.mock_api.get_events_by_year.return_value = [event_a] + self.mock_api.get_event_matches.return_value = [] + self.mock_api.get_event_teams.return_value = [] + + self.tba.fetch_all_matches(reset=True) + + call_kwargs = self.mock_api.get_event_matches.call_args + self.assertEqual(call_kwargs.kwargs.get('if_modified_since', ''), '') + + # ------------------------------------------------------------------ + # Test 5: previously cached matches for other events are preserved + # ------------------------------------------------------------------ + def test_previously_cached_matches_preserved_for_other_events(self): + event_old = _make_event('2026waold') + event_new = _make_event('2026wanew') + old_match = MagicMock() + self._initial_cache = { + 'events': [event_old], + 'matches': {'2026waold': [old_match]}, + 'event_teams': {}, + 'event_last_modified': {}, + 'headers': {}, + } + + self.mock_api.get_events_by_year.return_value = [event_new] + self.mock_api.get_event_matches.return_value = [] + self.mock_api.get_event_teams.return_value = [] + + result = self.tba.fetch_all_matches() + + # Old cached matches must be preserved. + self.assertIn('2026waold', result['matches']) + self.assertEqual(result['matches']['2026waold'], [old_match]) + # New event must also be present. + self.assertIn('2026wanew', result['matches']) + + # ------------------------------------------------------------------ + # Test 6: per-event if_modified_since is passed to the API + # ------------------------------------------------------------------ + def test_per_event_if_modified_since_used(self): + event_a = _make_event('2026waaaa') + stored_ts = 'Tue, 10 Jan 2026 08:00:00 GMT' + self._initial_cache = { + 'events': [event_a], + 'matches': {'2026waaaa': []}, + 'event_teams': {}, + 'event_last_modified': {'2026waaaa': stored_ts}, + 'headers': {}, + } + + self.mock_api.get_events_by_year.return_value = [event_a] + self.mock_api.get_event_matches.return_value = [] + self.mock_api.get_event_teams.return_value = [] + + self.tba.fetch_all_matches() + + call_kwargs = self.mock_api.get_event_matches.call_args + self.assertEqual(call_kwargs.kwargs.get('if_modified_since'), stored_ts) + + +if __name__ == '__main__': + unittest.main()