ansible multiprocessing

#!/usr/bin/env python

import subprocess
from multiprocessing import Pool
from functools import partial
import argparse
import sys
from itertools import chain

def execute_playbook(ansible_string, pb):
    # Note: If ansible fails, it prints to stdout
    # only system errors are handled by stderr
    print("Now executing playbook: {}".format(pb))
    ansible_string.append(pb)
    p = subprocess.Popen(ansible_string, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdout_output = None
    while not (stdout_output == '' and p.poll() is not None):
        stdout_output = p.stdout.readline()
        if stdout_output:
            # Stream output to stdout
            print(pb + ' | ' + stdout_output.strip())

    rc = p.poll()
    stdout, stderr = p.communicate()
    if stderr:
        print("[-] ERROR: {}".format(stderr))
    return (pb, rc)

def build_ansible_string(extra_vars):
    ansible_string = ['ansible-playbook', '-i', 'localhost,', '-c', 'local']
    for var in extra_vars:
        ansible_string.append('-e')
        ansible_string.append(var)
    return ansible_string

def main():
    parser = argparse.ArgumentParser(description='Process ansible playbooks')
    parser.add_argument('playbooks', metavar='PLAYBOOK', nargs='+',
                        help='The name of an Ansible playbook.')
    parser.add_argument('-e', '--extra_var', metavar='EXTRA_VAR', nargs="+",
                        help="The key=value extra argument", action="append")
    args = parser.parse_args()
    playbooks = args.playbooks
    extra_vars = args.extra_var
    extra_vars = list(chain.from_iterable(extra_vars))
    ansible_string = build_ansible_string(extra_vars)

    num_of_playbooks = len(playbooks)
    pool = Pool()
    print("Number of playbooks to process: {}".format(num_of_playbooks))
    func = partial(execute_playbook, ansible_string)
    return_codes = pool.map(func, playbooks)
    pool.close()
    pool.join()
    print ('-' * 80)
    fail_counter = 0
    for playbook, rc in return_codes:
        if rc != 0:
            print("[-] {} failed to execute properly".format(playbook))
            fail_counter += 1

    if fail_counter != 0:
        print("[-] ERROR: Failed to run all playbooks successfully.")
        sys.exit(255)

    for pb, rc in return_codes:
        print("Playbook executed: {}".format(pb))
    print("[+] SUCCESS: All playbooks executed successfully.")
    sys.exit(0)

if __name__ == '__main__':
    main()

Usage

usage: execute.py [-h] [-e EXTRA_VAR [EXTRA_VAR ...]] PLAYBOOK [PLAYBOOK ...]

Process ansible playbooks

positional arguments:
  PLAYBOOK              The name of an Ansible playbook.

optional arguments:
  -h, --help            show this help message and exit
  -e EXTRA_VAR [EXTRA_VAR ...], --extra_var EXTRA_VAR [EXTRA_VAR ...]
                        The key=value extra argument

Sample output

Number of playbooks to process: 2
Now executing playbook: tests/date1.yml
Now executing playbook: tests/date2.yml
tests/date1.yml |
tests/date2.yml |
tests/date2.yml | PLAY [127.0.0.1] ***************************************************************
tests/date1.yml | PLAY [127.0.0.1] ***************************************************************
tests/date2.yml |
tests/date1.yml |
tests/date2.yml | TASK [setup] *******************************************************************
tests/date1.yml | TASK [setup] *******************************************************************
tests/date2.yml | ok: [localhost]
tests/date1.yml | ok: [localhost]
tests/date2.yml |
tests/date2.yml | TASK [Get the date] ************************************************************
tests/date1.yml |
tests/date1.yml | TASK [Get the date] ************************************************************
tests/date2.yml | changed: [localhost]
tests/date1.yml | changed: [localhost]
tests/date1.yml |
tests/date1.yml | TASK [Debug the date] **********************************************************
tests/date2.yml |
tests/date2.yml | TASK [Debug the date] **********************************************************
tests/date1.yml | ok: [localhost] => {
tests/date1.yml | "msg": [
tests/date1.yml | "Tue Dec 20 10:25:55 CST 2016"
tests/date1.yml | ]
tests/date1.yml | }
tests/date2.yml | ok: [localhost] => {
tests/date2.yml | "msg": [
tests/date2.yml | "Tue Dec 20 10:25:55 CST 2016"
tests/date2.yml | ]
tests/date2.yml | }
tests/date1.yml |
tests/date1.yml | PLAY RECAP *********************************************************************
tests/date1.yml | localhost                  : ok=3    changed=1    unreachable=0    failed=0
tests/date1.yml |
tests/date2.yml |
tests/date2.yml | PLAY RECAP *********************************************************************
tests/date2.yml | localhost                  : ok=3    changed=1    unreachable=0    failed=0
tests/date2.yml |
--------------------------------------------------------------------------------
Playbook executed: tests/date1.yml
Playbook executed: tests/date2.yml
[+] SUCCESS: All playbooks executed successfully.