rsync but with an overall progress indicator, based on the total number of files that needs to be transferred

rsync_with_overall_progress.py

#!/usr/local/bin/python3

# Usage: python3 rsync_with_overall_progress.py source destination

import subprocess, re, sys, os, argparse

try:
    import progressbar
    progressbar_module_exists = True
except ImportError:
    progressbar_module_exists = False

import re
import subprocess

def rsync_with_progress(source, destination):
    rsync = subprocess.Popen(['rsync', '-avP', source, destination],
                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=False)

    total_files = None
    files_copied = 0
    bar = None
    non_percent_line_encountered = False

    for line in iter(rsync.stdout.readline, b''):
        line = line.decode('ISO-8859-1')
        if total_files is None:
            match = re.search(r'(\d+) files to consider', line)
            if match:
                total_files = int(match.group(1))
                if progressbar_module_exists:
                    bar = progressbar.ProgressBar(max_value=total_files)
        elif '%' in line:
            if non_percent_line_encountered:
                files_copied += 1
                non_percent_line_encountered = False
            if progressbar_module_exists:
                bar.update(files_copied)
            else:
                print(f'\r{files_copied}/{total_files}', end='')
        else:
            non_percent_line_encountered = True

    if rsync.wait() != 0:
        print('\nrsync failed')
        return False
    else:
        print('\nrsync completed successfully')
        return True

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('source')
    parser.add_argument('destination')
    args = parser.parse_args()

    if not os.path.exists(args.source):
        print(f'Source {args.source} does not exist')
        sys.exit(1)

    if not os.path.exists(args.destination):
        print(f'Destination {args.destination} does not exist')
        sys.exit(1)

    print(f'Copying {args.source} to {args.destination}')
    if rsync_with_progress(args.source, args.destination):
        print(f'Finished copying {args.source} to {args.destination}')

if __name__ == '__main__':
    main()