minecraft_dl.py

#!/usr/bin/env python3
"""
Download all Minecraft versions' data for archival purposes
"""

import urllib.parse, urllib.request
import os
import json
from pathlib import Path

# BUF_SIZE = 16 * 1024

def save_file(url, path):
	"""
	Save a file at `url` to `path`
	"""
	
	tries = 0
	
	while tries < 3:
		try:
			urllib.request.urlretrieve(url, path)
			break
		except ConnectionResetError:
			print("Connection was reset by peer. Trying again...")
			tries += 1
		except urllib.error.URLError:
			print("URLError. Trying again...")
			tries += 1
	else:
		raise ConnectionResetError("Connection reset by peer")
	
# 	with urllib.request.urlopen(url) as req:
# 		with open(path, "wb") as f:
# 			while True:
# 				data = req.read(BUF_SIZE)
# 				
# 				if data:
# 					f.write(data)
# 				else:
# 					break

class Archive:
	def __init__(self, root):
		self.root = root
	
	def save(self, url, allow_cache=True):
		"""
		Download a file from the given URL, with the name being the path.
		"""
		
		parsed_url = urllib.parse.urlparse(url)
		download_path = f"{self.root}/{parsed_url.netloc}/{parsed_url.path}"
		
		# If this file already exists, we shouldn't need to download it again
		# unless it can possibly change.
		if allow_cache and os.path.exists(download_path):
			print(f"Won't redownload {download_path}")
			return
		else:
			print(f"Downloading {download_path}")
		
		# Make subdirectories if they don't already exist
		os.makedirs(Path(download_path).parent, exist_ok=True)
		
		# Download the file
		save_file(url, download_path)
	
	def remove(self, url):
		parsed_url = urllib.parse.urlparse(url)
		download_path = f"{self.root}/{parsed_url.netloc}/{parsed_url.path}"
		os.remove(download_path)
	
	def readJson(self, url):
		parsed_url = urllib.parse.urlparse(url)
		download_path = f"{self.root}/{parsed_url.netloc}/{parsed_url.path}"
		return json.loads(Path(download_path).read_text())

def download_version(archive, version):
	print(f"*** Process index for version {version['id']} ***")
	
	# Index
	archive.save(version["url"])
	index = archive.readJson(version["url"])
	
	# Jar Downloads
	for k, v in index["downloads"].items():
		archive.save(v["url"])
	
	# Lib downloads
	for lib in index["libraries"]:
		if "artifact" in lib["downloads"]:
			a = lib["downloads"]["artifact"]
			if "url" in a:
				archive.save(a["url"])
		
		if "classifiers" in lib["downloads"]:
			for k, v in lib["downloads"]["classifiers"].items():
				archive.save(v["url"])
	
	# Logging config (ig?)
	if "logging" in index:
		archive.save(index["logging"]["client"]["file"]["url"])
	
	# Asset index
	archive.save(index["assetIndex"]["url"])
	asset_index = archive.readJson(index["assetIndex"]["url"])
	
	for name, meta in asset_index["objects"].items():
		archive.save(f"https://resources.download.minecraft.net/{meta['hash'][:2]}/{meta['hash']}")

def remove_version(archive, version):
	print(f"*** Remove version {version['id']} ***")
	
	# Index
	index = archive.readJson(version["url"])
	archive.remove(version["url"])
	
	# Jar Downloads
	for k, v in index["downloads"].items():
		archive.remove(v["url"])

def main():
	archive = Archive("./mc_archive")
	
	# Master list of minecraft versions which can be downloaded
	dont_update = False
	archive.save("https://piston-meta.mojang.com/mc/game/version_manifest.json", dont_update)
	archive.save("https://piston-meta.mojang.com/mc/game/version_manifest_v2.json", dont_update)
	
	version_manifest = archive.readJson("https://piston-meta.mojang.com/mc/game/version_manifest.json")
	print(version_manifest)
	
	for version in version_manifest["versions"]:
		if version["type"] != "snapshot":
			download_version(archive, version)
		# else:
			# remove_version(archive, version)

if __name__ == "__main__":
	main()