Skip to content

Instantly share code, notes, and snippets.

@kiproping
Last active June 21, 2017 20:04
Show Gist options
  • Save kiproping/b49979b6d8e385a01030188af79cdd16 to your computer and use it in GitHub Desktop.
Save kiproping/b49979b6d8e385a01030188af79cdd16 to your computer and use it in GitHub Desktop.
Downloading content from a list of files, handles 404 and 500 errors in CalledProcessorError
from subprocess import *
import time
from random import randint
path = "songs.lst"
sleepaftersong = 10 #seconds sleep after each song
sleepaftermax = 600 #seconds sleep after max consecutive songs
maxdownload = 40 #Downloads before long wait
def main():
count = 0
url = ""
for url in loadFiles():
if (count > 0 and count%maxdownload == 0): #Skip waiting at 0
time.sleep(randomtime(sleepaftermax, sleepaftermax+400))
try :
check_output(["wget -c --user-agent='Mozilla/5.0 (Linux; U; Android-4.0.3; en-us; Galaxy Nexus Build/IML74K) AppleWebKit/535.7 (KHTML, like Gecko) CrMo/16.0.912.75 Mobile Safari/535.7' --content-disposition -P assets/ "+ url.rstrip("\n") ], shell=True)
time.sleep(randomtime(sleepaftersong, sleepaftersong+30))
except CalledProcessError as e:
print e
time.sleep(randomtime(sleepaftersong, sleepaftersong+30))
count += 1
def loadFiles():
f = open(path, "r")
return f
def randomtime(start, end):
return randint(start, end)
if __name__ == "__main__":
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment