import requests
from bs4 import BeautifulSoup
import pandas as pd
standing_url = "https://fbref.com/en/comps/9/Premier-League-Stats"
data = requests.get(standing_url)
soup = BeautifulSoup(data.text)
standing_table = soup.select('table.stats_table')[0]
links = standing_table.find_all('a')
links = [l.get("href") for l in links]
links = [l for l in links if '/squads' in l]
links
['/en/squads/b8fd03ef/Manchester-City-Stats', '/en/squads/18bb7c10/Arsenal-Stats', '/en/squads/19538871/Manchester-United-Stats', '/en/squads/b2b47a98/Newcastle-United-Stats', '/en/squads/822bd0ba/Liverpool-Stats', '/en/squads/d07537b9/Brighton-and-Hove-Albion-Stats', '/en/squads/8602292d/Aston-Villa-Stats', '/en/squads/361ca564/Tottenham-Hotspur-Stats', '/en/squads/cd051869/Brentford-Stats', '/en/squads/fd962109/Fulham-Stats', '/en/squads/47c64c55/Crystal-Palace-Stats', '/en/squads/cff3d9bb/Chelsea-Stats', '/en/squads/8cec06e1/Wolverhampton-Wanderers-Stats', '/en/squads/7c21e445/West-Ham-United-Stats', '/en/squads/4ba7cbea/Bournemouth-Stats', '/en/squads/e4a775cb/Nottingham-Forest-Stats', '/en/squads/d3fd31cc/Everton-Stats', '/en/squads/a2d435b3/Leicester-City-Stats', '/en/squads/5bfb9659/Leeds-United-Stats', '/en/squads/33c895d4/Southampton-Stats']
team_urls = [f"https://fbref.com{l}" for l in links]
team_urls
['https://fbref.com/en/squads/b8fd03ef/Manchester-City-Stats', 'https://fbref.com/en/squads/18bb7c10/Arsenal-Stats', 'https://fbref.com/en/squads/19538871/Manchester-United-Stats', 'https://fbref.com/en/squads/b2b47a98/Newcastle-United-Stats', 'https://fbref.com/en/squads/822bd0ba/Liverpool-Stats', 'https://fbref.com/en/squads/d07537b9/Brighton-and-Hove-Albion-Stats', 'https://fbref.com/en/squads/8602292d/Aston-Villa-Stats', 'https://fbref.com/en/squads/361ca564/Tottenham-Hotspur-Stats', 'https://fbref.com/en/squads/cd051869/Brentford-Stats', 'https://fbref.com/en/squads/fd962109/Fulham-Stats', 'https://fbref.com/en/squads/47c64c55/Crystal-Palace-Stats', 'https://fbref.com/en/squads/cff3d9bb/Chelsea-Stats', 'https://fbref.com/en/squads/8cec06e1/Wolverhampton-Wanderers-Stats', 'https://fbref.com/en/squads/7c21e445/West-Ham-United-Stats', 'https://fbref.com/en/squads/4ba7cbea/Bournemouth-Stats', 'https://fbref.com/en/squads/e4a775cb/Nottingham-Forest-Stats', 'https://fbref.com/en/squads/d3fd31cc/Everton-Stats', 'https://fbref.com/en/squads/a2d435b3/Leicester-City-Stats', 'https://fbref.com/en/squads/5bfb9659/Leeds-United-Stats', 'https://fbref.com/en/squads/33c895d4/Southampton-Stats']
Extracting match statistics using pandas and requests
team1_url = team_urls[0]
t1_data = requests.get(team1_url)
matches = pd.read_html(t1_data.text, match="Scores & Fixtures")
matches[0]
| Date | Time | Comp | Round | Day | Venue | Result | GF | GA | Opponent | xG | xGA | Poss | Attendance | Captain | Formation | Referee | Match Report | Notes | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 2022-07-30 | 17:00 | Community Shield | FA Community Shield | Sat | Neutral | L | 1.0 | 3.0 | Liverpool | NaN | NaN | 57.0 | NaN | Rúben Dias | 4-3-3 | Craig Pawson | Match Report | NaN |
| 1 | 2022-08-07 | 16:30 | Premier League | Matchweek 1 | Sun | Away | W | 2.0 | 0.0 | West Ham | 2.2 | 0.5 | 75.0 | 62443.0 | İlkay Gündoğan | 4-3-3 | Michael Oliver | Match Report | NaN |
| 2 | 2022-08-13 | 15:00 | Premier League | Matchweek 2 | Sat | Home | W | 4.0 | 0.0 | Bournemouth | 1.7 | 0.1 | 67.0 | 53453.0 | İlkay Gündoğan | 4-2-3-1 | David Coote | Match Report | NaN |
| 3 | 2022-08-21 | 16:30 | Premier League | Matchweek 3 | Sun | Away | D | 3.0 | 3.0 | Newcastle Utd | 2.1 | 1.8 | 69.0 | 52258.0 | İlkay Gündoğan | 4-3-3 | Jarred Gillett | Match Report | NaN |
| 4 | 2022-08-27 | 15:00 | Premier League | Matchweek 4 | Sat | Home | W | 4.0 | 2.0 | Crystal Palace | 2.2 | 0.1 | 74.0 | 53112.0 | Kevin De Bruyne | 4-2-3-1 | Darren England | Match Report | NaN |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 56 | 2023-05-21 | 16:00 | Premier League | Matchweek 37 | Sun | Home | W | 1.0 | 0.0 | Chelsea | 1.2 | 1.2 | 64.0 | 53490.0 | Kyle Walker | 3-4-3◆ | Michael Oliver | Match Report | NaN |
| 57 | 2023-05-24 | 20:00 | Premier League | Matchweek 32 | Wed | Away | D | 1.0 | 1.0 | Brighton | 1.8 | 2.2 | 60.0 | 31388.0 | İlkay Gündoğan | 4-3-3 | Simon Hooper | Match Report | NaN |
| 58 | 2023-05-28 | 16:30 | Premier League | Matchweek 38 | Sun | Away | L | 0.0 | 1.0 | Brentford | 1.6 | 1.3 | 65.0 | 17120.0 | Kyle Walker | 3-2-4-1 | John Brooks | Match Report | NaN |
| 59 | 2023-06-03 | 15:00 | FA Cup | Final | Sat | Home | W | 2.0 | 1.0 | Manchester Utd | NaN | NaN | 60.0 | 83179.0 | İlkay Gündoğan | 3-2-4-1 | Paul Tierney | Match Report | NaN |
| 60 | 2023-06-10 | 22:00 | Champions Lg | Final | Sat | Neutral | NaN | NaN | NaN | it Inter | NaN | NaN | NaN | NaN | NaN | NaN | NaN | Head-to-Head | NaN |
61 rows × 19 columns
Getting match Shooting stats
soup = BeautifulSoup(t1_data.text)
links = soup.find_all('a')
links = [l.get('href') for l in links if 'Scores & Fixtures' in l]
fixture_data = [f"https://fbref.com{l}" for l in links]
fixture_data
['https://fbref.com/en/squads/b8fd03ef/2022-2023/matchlogs/all_comps/schedule/Manchester-City-Scores-and-Fixtures-All-Competitions', 'https://fbref.com/en/squads/b8fd03ef/2022-2023/matchlogs/c9/schedule/Manchester-City-Scores-and-Fixtures-Premier-League', 'https://fbref.com/en/squads/b8fd03ef/2022-2023/matchlogs/c8/schedule/Manchester-City-Scores-and-Fixtures-Champions-League', 'https://fbref.com/en/squads/b8fd03ef/2022-2023/matchlogs/c514/schedule/Manchester-City-Scores-and-Fixtures-FA-Cup', 'https://fbref.com/en/squads/b8fd03ef/2022-2023/matchlogs/c690/schedule/Manchester-City-Scores-and-Fixtures-EFL-Cup', 'https://fbref.com/en/squads/b8fd03ef/2022-2023/matchlogs/c602/schedule/Manchester-City-Scores-and-Fixtures-FA-Community-Shield', 'https://fbref.com/en/squads/b8fd03ef/2022-2023/matchlogs/all_comps/schedule/Manchester-City-Scores-and-Fixtures-All-Competitions', 'https://fbref.com/en/squads/b8fd03ef/2022-2023/matchlogs/c9/schedule/Manchester-City-Scores-and-Fixtures-Premier-League', 'https://fbref.com/en/squads/b8fd03ef/2022-2023/matchlogs/c8/schedule/Manchester-City-Scores-and-Fixtures-Champions-League', 'https://fbref.com/en/squads/b8fd03ef/2022-2023/matchlogs/c514/schedule/Manchester-City-Scores-and-Fixtures-FA-Cup', 'https://fbref.com/en/squads/b8fd03ef/2022-2023/matchlogs/c690/schedule/Manchester-City-Scores-and-Fixtures-EFL-Cup', 'https://fbref.com/en/squads/b8fd03ef/2022-2023/matchlogs/c602/schedule/Manchester-City-Scores-and-Fixtures-FA-Community-Shield', 'https://fbref.com#all_matchlogs', 'https://fbref.comNone', 'https://fbref.com/en/squads/b8fd03ef/2022-2023/matchlogs/all_comps/schedule/Manchester-City-Scores-and-Fixtures-All-Competitions', 'https://fbref.com/en/squads/b8fd03ef/2022-2023/matchlogs/c9/schedule/Manchester-City-Scores-and-Fixtures-Premier-League', 'https://fbref.com/en/squads/b8fd03ef/2022-2023/matchlogs/c8/schedule/Manchester-City-Scores-and-Fixtures-Champions-League', 'https://fbref.com/en/squads/b8fd03ef/2022-2023/matchlogs/c514/schedule/Manchester-City-Scores-and-Fixtures-FA-Cup', 'https://fbref.com/en/squads/b8fd03ef/2022-2023/matchlogs/c690/schedule/Manchester-City-Scores-and-Fixtures-EFL-Cup', 'https://fbref.com/en/squads/b8fd03ef/2022-2023/matchlogs/c602/schedule/Manchester-City-Scores-and-Fixtures-FA-Community-Shield', 'https://fbref.com/en/matches/']
fixture_data_url = fixture_data[0]
fixture_data = requests.get(fixture_data_url)
soup = BeautifulSoup(fixture_data.text)
fixture_links = soup.find_all('a')
fixture_links = [l.get('href') for l in fixture_links]
fixture_links = [l for l in fixture_links if l and 'all_comps/shooting' in l]
shooting_data = requests.get(f"https://fbref.com{fixture_links[0]}")
shooting_stats = pd.read_html(shooting_data.text, match="Shooting")
shooting_stats = shooting_stats[0]
shooting_stats
| For Manchester City | ... | Standard | Expected | Unnamed: 25_level_0 | |||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| Date | Time | Comp | Round | Day | Venue | Result | GF | GA | Opponent | ... | Dist | FK | PK | PKatt | xG | npxG | npxG/Sh | G-xG | np:G-xG | Match Report | |
| 0 | 2022-07-30 | 17:00 | Community Shield | FA Community Shield | Sat | Neutral | L | 1 | 3 | Liverpool | ... | NaN | NaN | 0 | 0 | NaN | NaN | NaN | NaN | NaN | Match Report |
| 1 | 2022-08-07 | 16:30 | Premier League | Matchweek 1 | Sun | Away | W | 2 | 0 | West Ham | ... | 18.7 | 1.0 | 1 | 1 | 2.2 | 1.4 | 0.11 | -0.2 | -0.4 | Match Report |
| 2 | 2022-08-13 | 15:00 | Premier League | Matchweek 2 | Sat | Home | W | 4 | 0 | Bournemouth | ... | 17.5 | 0.0 | 0 | 0 | 1.7 | 1.7 | 0.09 | 1.3 | 1.3 | Match Report |
| 3 | 2022-08-21 | 16:30 | Premier League | Matchweek 3 | Sun | Away | D | 3 | 3 | Newcastle Utd | ... | 16.2 | 1.0 | 0 | 0 | 2.1 | 2.1 | 0.10 | 0.9 | 0.9 | Match Report |
| 4 | 2022-08-27 | 15:00 | Premier League | Matchweek 4 | Sat | Home | W | 4 | 2 | Crystal Palace | ... | 14.1 | 0.0 | 0 | 0 | 2.2 | 2.2 | 0.13 | 1.8 | 1.8 | Match Report |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 56 | 2023-05-21 | 16:00 | Premier League | Matchweek 37 | Sun | Home | W | 1 | 0 | Chelsea | ... | 18.8 | 0.0 | 0 | 0 | 1.2 | 1.2 | 0.08 | -0.2 | -0.2 | Match Report |
| 57 | 2023-05-24 | 20:00 | Premier League | Matchweek 32 | Wed | Away | D | 1 | 1 | Brighton | ... | 13.9 | 0.0 | 0 | 0 | 1.8 | 1.8 | 0.14 | -0.8 | -0.8 | Match Report |
| 58 | 2023-05-28 | 16:30 | Premier League | Matchweek 38 | Sun | Away | L | 0 | 1 | Brentford | ... | 16.8 | 0.0 | 0 | 0 | 1.6 | 1.6 | 0.11 | -1.6 | -1.6 | Match Report |
| 59 | 2023-06-03 | 15:00 | FA Cup | Final | Sat | Home | W | 2 | 1 | Manchester Utd | ... | NaN | NaN | 0 | 0 | NaN | NaN | NaN | NaN | NaN | Match Report |
| 60 | NaN | NaN | NaN | NaN | NaN | NaN | 43-10-7 | 150 | 46 | NaN | ... | 16.8 | 24.0 | 14 | 18 | 104.2 | 92.8 | 0.12 | 42.8 | 40.2 | NaN |
61 rows × 26 columns
droping the top level indexshooting_stats.columns = shooting_stats.columns.droplevel()
shooting_stats
| Date | Time | Comp | Round | Day | Venue | Result | GF | GA | Opponent | ... | Dist | FK | PK | PKatt | xG | npxG | npxG/Sh | G-xG | np:G-xG | Match Report | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 2022-07-30 | 17:00 | Community Shield | FA Community Shield | Sat | Neutral | L | 1 | 3 | Liverpool | ... | NaN | NaN | 0 | 0 | NaN | NaN | NaN | NaN | NaN | Match Report |
| 1 | 2022-08-07 | 16:30 | Premier League | Matchweek 1 | Sun | Away | W | 2 | 0 | West Ham | ... | 18.7 | 1.0 | 1 | 1 | 2.2 | 1.4 | 0.11 | -0.2 | -0.4 | Match Report |
| 2 | 2022-08-13 | 15:00 | Premier League | Matchweek 2 | Sat | Home | W | 4 | 0 | Bournemouth | ... | 17.5 | 0.0 | 0 | 0 | 1.7 | 1.7 | 0.09 | 1.3 | 1.3 | Match Report |
| 3 | 2022-08-21 | 16:30 | Premier League | Matchweek 3 | Sun | Away | D | 3 | 3 | Newcastle Utd | ... | 16.2 | 1.0 | 0 | 0 | 2.1 | 2.1 | 0.10 | 0.9 | 0.9 | Match Report |
| 4 | 2022-08-27 | 15:00 | Premier League | Matchweek 4 | Sat | Home | W | 4 | 2 | Crystal Palace | ... | 14.1 | 0.0 | 0 | 0 | 2.2 | 2.2 | 0.13 | 1.8 | 1.8 | Match Report |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 56 | 2023-05-21 | 16:00 | Premier League | Matchweek 37 | Sun | Home | W | 1 | 0 | Chelsea | ... | 18.8 | 0.0 | 0 | 0 | 1.2 | 1.2 | 0.08 | -0.2 | -0.2 | Match Report |
| 57 | 2023-05-24 | 20:00 | Premier League | Matchweek 32 | Wed | Away | D | 1 | 1 | Brighton | ... | 13.9 | 0.0 | 0 | 0 | 1.8 | 1.8 | 0.14 | -0.8 | -0.8 | Match Report |
| 58 | 2023-05-28 | 16:30 | Premier League | Matchweek 38 | Sun | Away | L | 0 | 1 | Brentford | ... | 16.8 | 0.0 | 0 | 0 | 1.6 | 1.6 | 0.11 | -1.6 | -1.6 | Match Report |
| 59 | 2023-06-03 | 15:00 | FA Cup | Final | Sat | Home | W | 2 | 1 | Manchester Utd | ... | NaN | NaN | 0 | 0 | NaN | NaN | NaN | NaN | NaN | Match Report |
| 60 | NaN | NaN | NaN | NaN | NaN | NaN | 43-10-7 | 150 | 46 | NaN | ... | 16.8 | 24.0 | 14 | 18 | 104.2 | 92.8 | 0.12 | 42.8 | 40.2 | NaN |
61 rows × 26 columns
Combining our mathes df and our shooting df
team_stats = matches[0].merge(shooting_stats[["Date", "Sh", "SoT", "Dist", "FK", "PK", "PKatt"]], on="Date")
team_stats.head()
| Date | Time | Comp | Round | Day | Venue | Result | GF | GA | Opponent | ... | Formation | Referee | Match Report | Notes | Sh | SoT | Dist | FK | PK | PKatt | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 2022-07-30 | 17:00 | Community Shield | FA Community Shield | Sat | Neutral | L | 1.0 | 3.0 | Liverpool | ... | 4-3-3 | Craig Pawson | Match Report | NaN | 14 | 8 | NaN | NaN | 0 | 0 |
| 1 | 2022-08-07 | 16:30 | Premier League | Matchweek 1 | Sun | Away | W | 2.0 | 0.0 | West Ham | ... | 4-3-3 | Michael Oliver | Match Report | NaN | 13 | 1 | 18.7 | 1.0 | 1 | 1 |
| 2 | 2022-08-13 | 15:00 | Premier League | Matchweek 2 | Sat | Home | W | 4.0 | 0.0 | Bournemouth | ... | 4-2-3-1 | David Coote | Match Report | NaN | 19 | 7 | 17.5 | 0.0 | 0 | 0 |
| 3 | 2022-08-21 | 16:30 | Premier League | Matchweek 3 | Sun | Away | D | 3.0 | 3.0 | Newcastle Utd | ... | 4-3-3 | Jarred Gillett | Match Report | NaN | 21 | 10 | 16.2 | 1.0 | 0 | 0 |
| 4 | 2022-08-27 | 15:00 | Premier League | Matchweek 4 | Sat | Home | W | 4.0 | 2.0 | Crystal Palace | ... | 4-2-3-1 | Darren England | Match Report | NaN | 18 | 5 | 14.1 | 0.0 | 0 | 0 |
5 rows × 25 columns
print(team_data.shape)
print(matches[0].shape)
print(shooting_stats.shape)
(60, 25) (61, 19) (61, 26)
Using Loops to query multiple seasons
years = list(range(2022, 2018, -1))
all_matches = []
years
[2022, 2021, 2020, 2019]
import time
for year in years:
data = requests.get(standing_url)
soup = BeautifulSoup(data.text)
standing_table = soup.select('table.stats_table')[0]
links = [l.get("href") for l in standing_table.find_all('a')]
links = [l for l in links if '/squads/' in l]
team_urls = [f"https://fbref.com{l}" for l in links]
previous_season = soup.select("a.prev")[0].get("href")
standings_url = f"https://fbref.com{previous_season}"
# scraping through each individual team logs
for team_url in team_urls:
#geting individual team names
team_name = team1_url.split("/")[-1].replace("-Stats", "").replace("-", " ")
#getting individual team match stats
data = requests.get(team_url)
matches = pd.read_html(data.text, match="Scores & Fixtures")[0]
#getting shooting stats
soup = BeautifulSoup(data.text)
links = [l.get("href") for l in soup.find_all('a')]
links = [l for l in links if l and 'all_comps/shooting/' in l]
data = requests.get(f"https://fbref.com{links[0]}")
shooting = pd.read_html(data.text, match="Shooting")[0]
shooting.columns = shooting.columns.droplevel()
##############################################################################################
# when mergering the two tables some of the shooting data in some teams is not available so ##
# to overcome that, we have to use try-except statements ##
##############################################################################################
try:
epl_team_data = matches.merge(shooting[["Date", "Sh", "SoT", "Dist", "FK", "PK", "PKatt"]], on="Date")
except ValueError:
continue
epl_team_data = epl_team_data[epl_team_data["Comp"]=="Premier League"]
epl_team_data["season"] = year
epl_team_data["Team"] = team_name
all_matches.append(epl_team_data)
time.sleep(1)
len(all_matches)
80
epl_matches = pd.concat(all_matches)
epl_matches
| Date | Time | Comp | Round | Day | Venue | Result | GF | GA | Opponent | ... | Match Report | Notes | Sh | SoT | Dist | FK | PK | PKatt | season | Team | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 1 | 2018-08-12 | 16:00 | Premier League | Matchweek 1 | Sun | Away | W | 2 | 0 | Arsenal | ... | Match Report | NaN | 17.0 | 8.0 | 18.2 | 1.0 | 0 | 0 | 2022 | Manchester City |
| 2 | 2018-08-19 | 13:30 | Premier League | Matchweek 2 | Sun | Home | W | 6 | 1 | Huddersfield | ... | Match Report | NaN | 32.0 | 14.0 | 18.0 | 1.0 | 0 | 0 | 2022 | Manchester City |
| 3 | 2018-08-25 | 12:30 | Premier League | Matchweek 3 | Sat | Away | D | 1 | 1 | Wolves | ... | Match Report | NaN | 18.0 | 6.0 | 18.6 | 1.0 | 0 | 0 | 2022 | Manchester City |
| 4 | 2018-09-01 | 17:30 | Premier League | Matchweek 4 | Sat | Home | W | 2 | 1 | Newcastle Utd | ... | Match Report | NaN | 24.0 | 8.0 | 17.5 | 3.0 | 0 | 0 | 2022 | Manchester City |
| 5 | 2018-09-15 | 15:00 | Premier League | Matchweek 5 | Sat | Home | W | 3 | 0 | Fulham | ... | Match Report | NaN | 28.0 | 9.0 | 16.6 | 1.0 | 0 | 0 | 2022 | Manchester City |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 35 | 2019-04-13 | 12:30 | Premier League | Matchweek 34 | Sat | Away | L | 0 | 4 | Tottenham | ... | Match Report | NaN | 7.0 | 1.0 | 18.9 | 1.0 | 0 | 0 | 2019 | Manchester City |
| 36 | 2019-04-20 | 15:00 | Premier League | Matchweek 35 | Sat | Home | L | 1 | 2 | Watford | ... | Match Report | NaN | 13.0 | 3.0 | 18.1 | 1.0 | 0 | 0 | 2019 | Manchester City |
| 37 | 2019-04-26 | 20:00 | Premier League | Matchweek 36 | Fri | Away | L | 0 | 5 | Liverpool | ... | Match Report | NaN | 5.0 | 1.0 | 21.6 | 0.0 | 0 | 0 | 2019 | Manchester City |
| 38 | 2019-05-05 | 14:00 | Premier League | Matchweek 37 | Sun | Home | D | 1 | 1 | Manchester Utd | ... | Match Report | NaN | 7.0 | 3.0 | 19.0 | 1.0 | 0 | 0 | 2019 | Manchester City |
| 39 | 2019-05-12 | 15:00 | Premier League | Matchweek 38 | Sun | Away | D | 1 | 1 | Southampton | ... | Match Report | NaN | 10.0 | 3.0 | 22.2 | 0.0 | 0 | 0 | 2019 | Manchester City |
3040 rows × 27 columns
epl_matches.columns = [c.lower() for c in epl_matches.columns]
epl_matches.to_csv("epl_matches(season from 2019-2022)")