Skip to content

Instantly share code, notes, and snippets.

@prot0man

prot0man/jenkins_downloader.py

Last active January 28, 2020 22:56
Show Gist options
  • Save prot0man/86f11d99efb17118c4e912c3bb11b4f1 to your computer and use it in GitHub Desktop.
Save prot0man/86f11d99efb17118c4e912c3bb11b4f1 to your computer and use it in GitHub Desktop.
Download ZIP
A Jenkins plugin and dependency downloader for offline Jenkins instances
Raw
jenkins_downloader.py
"""
Attempts to download the specified jenkins plugin and all of its dependencies.
"""
import subprocess
import os
import re
import requests
import argparse
import logging
import zipfile
MANIFEST_PATH = "META-INF/MANIFEST.MF"
PLUGIN_DEP_RESTR = "Plugin\-Dependencies:\s*(.*?)\s+"
PLUGIN_DEP_RE = re.compile(PLUGIN_DEP_RESTR, re.MULTILINE)
UPDATE_URL = "https://updates.jenkins.io/download/plugins"
LATEST_URL = "https://updates.jenkins-ci.org/latest"
INSTALLED = []
INSTALL_FRESH = False
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("jenkins-plugin-downloader")
def exec_cmd(cmd,cwd=None):
p = subprocess.Popen(cmd, shell=True,stdin=subprocess.PIPE, stdout=subprocess.PIPE, cwd=cwd)
stdout, stderr = p.communicate()
return p.returncode, stdout, stderr
def get_latest_plugin_url(plugin_name):
plugin_url ="{0}/{1}.hpi".format(LATEST_URL, plugin_name)
return plugin_url
def get_plugin_url(plugin_name, plugin_version):
plugin_url ="{0}/{1}/{2}/{1}.hpi".format(UPDATE_URL, plugin_name, plugin_version)
return plugin_url
def get_plugin_manifest_data(plugin_path):
# get the contents of the manifest file
try:
zf = zipfile.ZipFile(plugin_path)
except Exception as e:
# if we were unable to unzip and it was an html file, then it's
# most likely a 404 not found.
with open(plugin_path, "r") as hfile:
contents = hfile.read()
if contents.find("<html>") != -1:
errmsg = "Couldn't find plugin %s on %s" % (plugin_path, UPDATE_URL)
else:
errmsg = "Unable to unzip %s: %s" % (plugin_path, e)
raise Exception(errmsg)
manifest_data = None
if MANIFEST_PATH in zf.namelist():
manifest_data = zf.read(MANIFEST_PATH)
zf.close()
# if manifest_data was never set, then it wasn't in the zip. Bail.
if manifest_data is None:
errmsg = "Unable to find %s in plugin." % MANIFEST_PATH
raise Exception(errmsg)
# filter out '\r\n ' pattern which is inserted to make a value span multiple
# lines for pretty output so that we can get the actual values
manifest_data = manifest_data.replace("\r\n ", "")
return manifest_data
def get_plugin_deps_from_manifest(manifest_data):
mg = PLUGIN_DEP_RE.search(manifest_data)
plugins = []
if mg:
plugin_dep_str = mg.group(1)
# split the comma separated list of plugins
plugin_descriptors = plugin_dep_str.split(",")
# each of these plugins has contains the name, a colon, and the version.
for pd in plugin_descriptors:
# skip optional plugins...
if pd.find(";resolution") != -1:
# if this is tagged as optional, get pname/pversion portion of
# it.
pd = pd.split(";")[0]
try:
pname, pversion = pd.split(":")
except:
# this means we have a manifest we don't know how to handle
logger.error("!!UNABLE TO SPLIT %s" % pd)
logger.error("MANIFEST was %s" % manifest_data)
raise
plugins.append((pname, pversion))
else:
if manifest_data.find("Plugin-Dependencies") != -1:
errmsg = "Unable to parse plugin dependencies from manifest"
logger.error(errmsg)
logger.error(manifest_data)
return plugins
return plugins
def download_file(url, outpath):
r = requests.get(url, allow_redirects=True)
with open(outpath, "wb") as hfile:
hfile.write(r.content)
def download_plugin(plugin_name, plugin_version, outpath):
plugin_url = get_plugin_url(plugin_name, plugin_version)
logger.info("Downloading {0}({1}) to {2}".format(plugin_name, plugin_version, outpath))
download_file(plugin_url, outpath)
def download_latest_plugin(plugin_name, outpath):
plugin_url = get_latest_plugin_url(plugin_name)
logger.info("Downloading lateast %s" % plugin_url)
download_file(plugin_url, outpath)
def is_installed(pname, pversion, outpath):
if (pname, pversion) in INSTALLED:
return True
if not INSTALL_FRESH and os.path.exists(outpath):
# if we don't have to install fresh every time and it's downloaded, skip
return True
return False
def download_jenkins_plugin_dependencies(plugin_path, outdir=""):
if not os.path.exists(plugin_path):
raise Exception("File %s does not exist" % plugin_path)
if not outdir:
outdir = os.getcwd()
# get the data in the manifest file in the archive
manifest_data = get_plugin_manifest_data(plugin_path)
# parse out the the list of dependencies from the manifest
plugins = get_plugin_deps_from_manifest(manifest_data)
# download each dependency
for pname, pversion in plugins:
outfile = "{0}_{1}.hpi".format(pname, pversion)
outpath = os.path.join(outdir, outfile)
if is_installed(pname, pversion, outpath):
continue
INSTALLED.append((pname, pversion))
download_plugin(pname, pversion, outpath)
download_jenkins_plugin_dependencies(outpath, outdir)
return plugins
def download_jenkins_plugin(plugin_name, outdir):
outfile = "{0}.hpi".format(plugin_name)
outpath = os.path.join(outdir, outfile)
download_latest_plugin(plugin_name, outpath)
download_jenkins_plugin_dependencies(outpath, outdir)
return outpath
def parse_args():
parser = argparse.ArgumentParser(description="Jenkins Dependency Downloader")
parser.add_argument("plugin_name", help="The name of the plugin to download")
parser.add_argument("-o", "--outdir", dest="outdir",
help="The directory to store the plugin and dependencies in")
args = parser.parse_args()
return args
if __name__ == "__main__":
args = parse_args()
plugin_name = args.plugin_name
outdir = args.outdir
if not outdir:
outdir = os.path.abspath("outdir")
if not os.path.exists(outdir):
os.makedirs(outdir)
download_jenkins_plugin(args.plugin_name, outdir)
logging.info("Stored plugins in %s" % outdir)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment