cache-warmup.py

import pandas as pd
import requests, csv, time, lxml, cchardet, re
from bs4 import BeautifulSoup
import concurrent.futures

sitemap = '' # Sitemap URL

s = requests.session()
sm = s.get(sitemap)
sm_soup = BeautifulSoup(sm.text, 'lxml')

links = [x.text for x in sm_soup.find_all('loc')]
print('Found {} links'.format(len(links)))

i = 0
failed_urls = []
success_urls = []

print('{} URLs already warmed up'.format(len(success_urls)))
for link in links:

    if i == 0:
        last_time = time.time()

    if i % 100 == False:
        print(" - {} completed -- took {} seconds".format(i, round(time.time()-last_time)))
        last_time = time.time()

    if link in success_urls:
        continue

    req = s.get(link)

    if req.status_code == 200:
        success_urls.append(link)
        time.sleep(0.2)
    else:
        failed_urls.append(link)

    i = i+1


if len(failed_urls) > 0:
    
    print('')
    print('')
    print("{} URLs failed: ")
    for x in failed_urls:
        print(" - {}".format(x))