Web-scraping Hindi (Bollywood) movies from IMDb

from requests import get 
import numpy as np 
import pandas as pd 
from bs4 import BeautifulSoup
import time as time 
from tqdm import tqdm

from IPython.core.display import clear_output

names, year, imdb_rating, metascore, num_votes = [], [], [], [], [] 

start_time = time.time()
requests = 0

years_url = [str(i) for i in range(1950,2006)]
page_iter = [0, 51, 101, 151, 201]

for year_url in tqdm(years_url):
    for page_num in tqdm(page_iter):
        #URL to parse 
        url = 'https://www.imdb.com/search/title/?title_type=feature,&release_date={0},{0}&countries=in&languages=hi&sort=num_votes,desc&start={1}&ref_=adv_prv'.format(int(year_url), int(page_num))
        response = get(url)
        
        #Sleep to carve out load 
        time.sleep(np.random.randint(1,5))
        
        #Estimate time elapsed per request
        requests += 1
        elapsed_time = time.time() - start_time
        print('Request:{}; Frequency: {} requests/s'.format(requests, requests/elapsed_time))
        clear_output(wait = True)
        
        html_soup = BeautifulSoup(response.text, 'html.parser')
        movie_containers = html_soup.find_all('div', class_='lister-item mode-advanced')
        
        for i, container in enumerate(movie_containers):
            container_entry = movie_containers[i] 
            movie_name = container_entry.h3.a.text
            names.append(movie_name)
            
            movie_year = container_entry.h3.find('span',class_='lister-item-year text-muted unbold').text.strip('()')
            year.append(movie_year)
            #print(movie_name, movie_year)
            
            try:
                movie_rating = float(container_entry.strong.text)
                imdb_rating.append(movie_rating)
            except AttributeError:
                imdb_rating.append(np.nan)
            
            try:
                movie_votes = float(''.join(container_entry.find('span', attrs = {'name':'nv'}).text.split(',')))
                num_votes.append(movie_votes)
            except (AttributeError, ValueError):
                num_votes.append(np.nan)
                
            try:
                movie_metascore = float(container_entry.find('span', class_='metascore').text.strip())
                metascore.append(movie_metascore)
            except AttributeError:
                metascore.append(np.nan)
    
    print('Making dataframe for year {}'.format(year_url))
    df_movies = pd.DataFrame({'name':names,'year':year,'rating':imdb_rating,'metascore':metascore,'num_votes':num_votes})
    df_movies.to_csv('./temp_imdb_files/bollywood_data_{}.csv'.format(year_url),sep=',',header=True, index=False)
    del df_movies

100%|██████████| 5/5 [00:30<00:00,  6.14s/it]
100%|██████████| 56/56 [20:00<00:00, 21.44s/it]

Making dataframe for year 2005

df1 = pd.read_csv('./temp_imdb_files/bollywood_data_2005.csv',sep=',')
df2 = pd.read_csv('./temp_imdb_files/bollywood_data_2020.csv',sep=',')

df3 = pd.concat((df1, df2)).reset_index(drop=True)

df3.to_csv('./bollywood_movies_data_1950_2020_new.csv',sep=',',header=True, index=False)

df3.year.value_counts()

2004           249
2001           249
2005           248
2000           246
1991           241
              ... 
II) (1988        1
II) (1957        1
XVII) (2016      1
IV) (2011        1
I) (1954         1
Name: year, Length: 181, dtype: int64