nba_playoffs_game_updater.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335
  1. #!/usr/bin/env python3
  2. import gspread
  3. import json
  4. import random
  5. from os import sys
  6. from oauth2client.service_account import ServiceAccountCredentials
  7. import datetime
  8. import time
  9. from nba_api.stats.static import players
  10. from nba_api.stats.endpoints import playergamelog
  11. import timeout_decorator
  12. import urllib
  13. try:
  14. from nba_api.library.debug.debug import DEBUG_STORAGE
  15. except ImportError:
  16. DEBUG_STORAGE = False
  17. #spreadsheet_key = '1QBQvGSMesox1gwjpaoK-0-p3n-c4I_L73PWCggjdayM' # 2019 Official
  18. #spreadsheet_key = '14pHOScaGXvN83iCca6_5p6QoViYvo223cIJD9nnl7TI' # 2019 Test
  19. #spreadsheet_key = '1n2qAxDhy3B-a20cn92H340GoPeKQE8fpztPlzKpGw80' # 2020 Test
  20. #spreadsheet_key = '1ajlHmH-dUzwkVfD-4NgpkK8ni3I3UuUFcdefF_DUOyg' # 2020 Official
  21. #spreadsheet_key = '1FgoBfPw4Vhi89rcYgxnAxLTeJ84GLXD1r39K0DXqxq4' # 2021 Official
  22. #spreadsheet_key = '1Gt4J1nNOv1E3-gikopSJQCD7nAqx7DEYxUqsZNac9_Y' # 2022 Official
  23. #spreadsheet_key = '1taZ97_PKEGYdSnMAeraPAbFR2U_7-P1GamzIdhlVXjc' # 2023 Test
  24. spreadsheet_key = '10qAZWyRhwo4Hb4_5X0Q4LFFSudGn2okJNLojwGAg0yI' # 2023 Official
  25. json_keyfile = 'NBA Playoffs Game-1f9a46f0715c.json'
  26. day = 'today' # today
  27. #day = datetime.date(2023, 4, 27) # set date manually
  28. nba_cooldown = random.gammavariate(alpha=9, beta=0.4) # don't hammer the NBA.com API
  29. stats=['PTS', 'REB', 'AST', 'STL', 'BLK', 'TOV', 'WL'] # stats appear in this order
  30. STATS_HEADERS = {
  31. 'Host': 'stats.nba.com',
  32. 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:72.0) Gecko/20100101 Firefox/72.0',
  33. 'Accept': 'application/json, text/plain, */*',
  34. 'Accept-Language': 'en-US,en;q=0.5',
  35. 'Accept-Encoding': 'gzip, deflate, br',
  36. 'x-nba-stats-origin': 'stats',
  37. 'x-nba-stats-token': 'true',
  38. 'Connection': 'keep-alive',
  39. 'Referer': 'https://stats.nba.com/',
  40. 'Pragma': 'no-cache',
  41. 'Cache-Control': 'no-cache',
  42. }
  43. # Proxy URLs
  44. #proxy_url="https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt"
  45. proxy_url="https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt"
  46. # Manual list of proxies
  47. proxies = []
  48. ###############################################################################
  49. def buildProxyList(proxies=[], raw_text_url="https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt"):
  50. good_proxy_list = []
  51. proxy_list = []
  52. r = urllib.request.urlopen(raw_text_url)
  53. for line in r:
  54. line = line.decode("utf-8")
  55. line = line.strip()
  56. proxy_list.append(line)
  57. random.shuffle(proxy_list)
  58. proxy_list = proxies + proxy_list
  59. return proxy_list, good_proxy_list
  60. """
  61. Returns a worksheet instance
  62. """
  63. def getWorksheet(spreadsheet_key, json_keyfile):
  64. try:
  65. scope = ['https://spreadsheets.google.com/feeds',
  66. 'https://www.googleapis.com/auth/drive']
  67. credentials = ServiceAccountCredentials.from_json_keyfile_name(json_keyfile, scope)
  68. gc = gspread.authorize(credentials)
  69. spreadsheet = gc.open_by_key(spreadsheet_key)
  70. worksheet = spreadsheet.worksheet("Picks")
  71. except Exception as e:
  72. print(f"Exception: {str(e)}")
  73. print("Could not retrieve worksheet!")
  74. print("Check your API key, credentials, or network!")
  75. raise(e)
  76. return worksheet
  77. """
  78. Returns a list of lists containing the values of all cells in the worksheet by row
  79. """
  80. def getAllValues(worksheet):
  81. return worksheet.get_all_values()
  82. """
  83. Create various date variables based on "today's" day
  84. """
  85. def setDates(day):
  86. if day == 'today':
  87. # in case games go past midnight
  88. date = datetime.datetime.now() - datetime.timedelta(hours=3)
  89. date = date.date()
  90. else:
  91. date = day
  92. url_date = date.strftime('%m/%d/%Y')
  93. year = date.year
  94. season = f"{format(str(year - 1))}-{str(year)[2:]}"
  95. return url_date, season, date
  96. """
  97. Determines the number of players in the pool
  98. """
  99. def getNumberOfParticipants(all_values):
  100. count=0
  101. for row_num, row in enumerate(all_values):
  102. if row[0] != "" and row_num >= 3 and count == 0:
  103. start=row_num
  104. count+=1
  105. elif row[0] != "" and row_num >= 3 and count == 1:
  106. end=row_num
  107. break
  108. num_participants = end - start
  109. return num_participants
  110. """
  111. Determines the active day's first and last rows
  112. """
  113. def getFirstRowLastRow(all_values, num_participants, current_date):
  114. first_row = None
  115. last_row = None
  116. for row_num, row in enumerate(all_values, start=1):
  117. date=row[0]
  118. if date != "" and row_num >= 4:
  119. day = datetime.datetime.strptime('{} {}'.format(date,
  120. str(current_date.year)),
  121. '%A, %B %d %Y')
  122. if day.date() == current_date:
  123. first_row = row_num
  124. last_row = first_row + num_participants - 1
  125. break
  126. return first_row, last_row
  127. """
  128. Rudimentary way to reduce player name errors
  129. """
  130. def cleanFirstNameLastName(player):
  131. first_name_last_name = player.split()
  132. first_name = first_name_last_name[0]
  133. first_name = first_name.replace('.', '')
  134. # New nickname for T.J. Warren should be "The Outlier"
  135. if first_name == "TJ":
  136. first_name = "T.J."
  137. elif first_name == "Donavan":
  138. first_name = "Donovan"
  139. elif first_name == "Domantis":
  140. first_name = "Domantas"
  141. last_name = first_name_last_name[1]
  142. player_clean = first_name + ' ' + last_name
  143. return player_clean
  144. """
  145. Create a unique list of players that have been selected today
  146. Also, append misspelled players to batch_update_list to autofix on next push if we can
  147. """
  148. def cleanPlayers(all_values, first_row, last_row, batch_update_list):
  149. players_unique = []
  150. for row_num, row in enumerate(all_values, start=1):
  151. if first_row <= row_num <= last_row:
  152. player = row[2]
  153. if player[-7:] != "-FIX!!!" and player != "":
  154. if len(players.find_players_by_full_name(player)) > 0:
  155. players_unique.append(player)
  156. else:
  157. player_clean = cleanFirstNameLastName(player)
  158. if len(players.find_players_by_full_name(player_clean)) > 0:
  159. all_values[row_num - 1][2] = player_clean
  160. batch_update_list.append({'range': f'{indexToLetter(2)}{row_num}', 'values': [[player_clean]]})
  161. players_unique.append(player_clean)
  162. else:
  163. print(f"Player: {player} not found, please fix name!")
  164. players_unique = list(dict.fromkeys(players_unique))
  165. return players_unique, batch_update_list, all_values
  166. """
  167. Pull player's gamelog from stats.nba.com based on the url_date and player_id
  168. """
  169. #@timeout_decorator.timeout(30)
  170. def getStats(players_unique, url_date, season, proxy_list=[], good_proxy_list=[]):
  171. stats_dict = {}
  172. for player in players_unique:
  173. player_info = players.find_players_by_full_name(player)
  174. player_id = player_info[0].get('id')
  175. print(f'Retrieving stats for: {player}')
  176. while True:
  177. # Move working proxies to the front of the list
  178. if len(good_proxy_list) > 0:
  179. proxy_list = good_proxy_list + proxy_list
  180. # Remove duplicate proxies
  181. proxy_list = list(dict.fromkeys(proxy_list))
  182. # Use the first proxy in the list
  183. request_proxy = proxy_list[0]
  184. try:
  185. print(f'Proxy: http://{request_proxy}')
  186. player_game_log = playergamelog.PlayerGameLog( player_id=player_id,
  187. proxy='http://' + request_proxy,
  188. season=season,
  189. timeout=10,
  190. league_id_nullable='00',
  191. season_type_all_star='Playoffs',
  192. date_from_nullable=url_date,
  193. date_to_nullable=url_date,
  194. )
  195. print('Success!')
  196. if request_proxy not in good_proxy_list:
  197. good_proxy_list.append(request_proxy)
  198. player_game_log_dict = player_game_log.get_dict()
  199. if DEBUG_STORAGE is False:
  200. time.sleep(nba_cooldown)
  201. break
  202. except OSError as e:
  203. print(e)
  204. if request_proxy in good_proxy_list:
  205. good_proxy_list.remove(request_proxy)
  206. else:
  207. print(f'Proxy refused, removing {request_proxy}')
  208. proxy_list.remove(request_proxy)
  209. continue
  210. except Exception as e:
  211. print(e)
  212. print('Could not connect to the NBA API, sleeping for 30 seconds')
  213. time.sleep(30)
  214. player_game_log_results = player_game_log_dict.get('resultSets')[0]
  215. player_game_log_headers = player_game_log_results.get('headers')
  216. # if player has no stats for this day, list will be empty
  217. if len(player_game_log_results.get('rowSet')) < 1:
  218. player_stats_dict = None
  219. else:
  220. player_game_log_stats = player_game_log_results.get('rowSet')[0]
  221. player_stats_dict = dict(zip(player_game_log_headers, player_game_log_stats))
  222. stats_dict[player] = player_stats_dict
  223. return stats_dict, good_proxy_list
  224. """
  225. Append stat cells that have changes to batch_update_list
  226. Also append player cells that need fixing to batch_update_list
  227. """
  228. def cellsToUpdate(all_values, first_row, last_row, stats_dict, stats, batch_update_list):
  229. for row_num, row in enumerate(all_values, start=1):
  230. if first_row <= row_num <= last_row:
  231. player_name = row[2]
  232. if player_name[-7:] != "-FIX!!!" and player_name in stats_dict.keys():
  233. if stats_dict[player_name] is not None:
  234. player_stats = stats_dict[player_name]
  235. if player_stats == "Fix!":
  236. batch_update_list.append({'range': f'{indexToLetter(2)}{row_num}', 'values': [[f'{player_name}-FIX!!!']]})
  237. continue
  238. for col_num, stat in enumerate(stats, start=3):
  239. pass
  240. #print(player_name, player_stats[stat])
  241. #print(player_name, f'{indexToLetter(col_num)}{row_num}', str(row[col_num]), f',', player_stats[stat])
  242. if str(player_stats[stat]) != str(row[col_num]) and player_stats[stat] is not None:
  243. #print('Update:', row_num, col_num, player_name, f'{indexToLetter(col_num)}{row_num}', str(row[col_num]), player_stats[stat])
  244. batch_update_list.append({'range': f'{indexToLetter(col_num)}{row_num}', 'values': [[f'{player_stats[stat]}']]}.copy())
  245. return batch_update_list
  246. """
  247. Convert zero-indexed column number to the appropriate column letter (A=0, B=1, C=2...)
  248. """
  249. def indexToLetter(index):
  250. return chr(ord('@')+int(index)+1)
  251. """
  252. Push changes to Google Sheet
  253. """
  254. def batchUpdate(batch_update_list):
  255. if len(batch_update_list) > 1:
  256. worksheet.batch_update(batch_update_list, value_input_option="USER_ENTERED")
  257. else:
  258. print('No update needed, sleeping for 1 minute')
  259. time.sleep(60)
  260. if __name__ == "__main__":
  261. # Use a combination of our good proxies with some fetched from the internet for variation
  262. proxy_list, good_proxy_list = buildProxyList(proxies=proxies, raw_text_url=proxy_url)
  263. while True:
  264. try:
  265. batch_update_list = []
  266. worksheet = getWorksheet(spreadsheet_key, json_keyfile)
  267. url_date, season, date = setDates(day)
  268. print("Date: " + str(date))
  269. all_values = getAllValues(worksheet)
  270. num_participants = getNumberOfParticipants(all_values)
  271. first_row, last_row = getFirstRowLastRow(all_values, num_participants, date)
  272. if first_row is None:
  273. print("No games today! Pausing for 1000 seconds...")
  274. time.sleep(1000)
  275. continue
  276. players_unique, batch_update_list, all_values = cleanPlayers(all_values, first_row, last_row, batch_update_list)
  277. stats_dict, good_proxy_list = getStats(players_unique, url_date, season, proxy_list=proxy_list, good_proxy_list=good_proxy_list)
  278. batch_update_list = cellsToUpdate(all_values, first_row, last_row, stats_dict, stats, batch_update_list)
  279. if len(batch_update_list) > 1:
  280. print(batch_update_list)
  281. batchUpdate(batch_update_list)
  282. except Exception as e:
  283. print(e)
  284. print('Sleeping for 10 seconds')
  285. time.sleep(10)
  286. continue