s3_updater.py

# Use Boto 3 and Botocore modules
import sys, threading, botocore
import boto3
from boto3.s3.transfer import TransferConfig

# Create Connection to S3
s3_client = boto3.client('s3')

# Create Connection to S3
s3_bucket = boto3.resource('s3')

# Turn up the bandwidth
config = TransferConfig(max_concurrency=500)

# Track Progress
class ProgressPercentage(object):
    def __init__(self, filename):
        self._filename = filename
        self._seen_so_far = 0
        self._lock = threading.Lock()
    def __call__(self, bytes_amount):
        # To simplify we'll assume this is hooked up
        # to a single filename.
        with self._lock:
            self._seen_so_far += bytes_amount
            sys.stdout.write(
                "\r%s --> %s bytes transferred" % (
                    self._filename, self._seen_so_far))
            sys.stdout.flush()

# Iterate over Buckets and Keys
bucket = s3_bucket.Bucket('bucket name')
exists = True
try:
    s3_bucket.meta.client.head_bucket(Bucket='bucket name')
except botocore.exceptions.ClientError as e:
    # If a client error is thrown, then check that it was a 404 error.
    # If it was a 404 error, then the bucket does not exist.
    error_code = int(e.response['Error']['Code'])
    if error_code == 404:
        exists = False

for key in bucket.objects.filter(Prefix='assets/'):
    copy_source = {
        'Bucket': key.bucket_name,
        'Key': key.key
    }
    s3_client.copy(
        copy_source, key.bucket_name, key.key,
        ExtraArgs={
            "Metadata": {
                "Cache-Control": "86400"
            },
            "MetadataDirective": "REPLACE"
        },
        Config=config,
        Callback=ProgressPercentage(key.key)
    )