extract_recommender.py

import requests
import bs4
import json
import click



@click.command()
@click.option('--perm', required=True, type=click.File('r'))
@click.option('--output', required=True, type=click.File('w'), default='-')
def main(perm, output):
    page = requests.get('https://cloud.google.com/recommender/docs/recommenders').text
    soup = bs4.BeautifulSoup(page)
    rows = soup.find_all('table')[0].find_all('tr')[1:]

    perms = [r for r in json.load(perm)['permissions'] if r.startswith('recommender') and 'Recommendations' in r]

    recommenders = []
    for r in rows:
        data = [td.text.strip() for td in r.find_all('td')]
        if len(data) == 4:
            category = data.pop(0)

        rec = dict(zip(('name', 'id', 'description'), data))
        rec['category'] = category

        if '\n' in rec['name']:
            rec['name'] = rec['name'].split('\n')[0]

        rec['permissions'] = get_perms(rec, perms)
        recommenders.append(rec)
    import pdb; pdb.set_trace()
    output.write(json.dumps(recommenders, indent=2))

def get_perms(rec, perms):

    id_parts = rec['id'].split('.')
    id = ''.join(id_parts[1:3]).lower()
    candidates = []
    for p in perms:
        if id in p.lower():
            candidates.append(p)
    print("%s -> %s" % (rec['id'], candidates))
    return candidates
            

if __name__ == '__main__':
    main()