minio-boto3-multipart.py

minio-boto3-multipart.py

import boto3
from boto3.s3.transfer import TransferConfig
import os

# MinIO configuration
minio_endpoint = "http://localhost:22000"  # Change to your MinIO endpoint
access_key = "minio"
secret_key = "minio123"
bucket_name = "test-bucket"
object_name = "1GB.zip"  # The path/key where the file will be stored in the bucket
file_path = "/home/prakash/Downloads/test-all-file-types/Sizes/1GB.zip"  # Path to the file you want to upload

# Configure the boto3 S3 client for MinIO (non-HTTPS)
s3_client = boto3.client(
    "s3",
    endpoint_url=minio_endpoint,  # Custom endpoint for MinIO
    aws_access_key_id=access_key,
    aws_secret_access_key=secret_key,
    config=boto3.session.Config(signature_version="s3v4"),  # Required for MinIO compatibility
)

# TransferConfig settings for multipart upload
# Adjust multipart_threshold and multipart_chunksize for your use case
config = TransferConfig(
    multipart_threshold=5 * 1024 * 1024,  # 5 MB threshold for multipart upload
    multipart_chunksize=5 * 1024 * 1024,  # Each part is 5 MB
    max_concurrency=4,                    # Number of threads for parallel uploads
    use_threads=True                      # Enable multi-threading
)

# Perform the multipart upload
try:
    response = s3_client.upload_file(
        Filename=file_path,
        Bucket=bucket_name,
        Key=object_name,
        Config=config
    )
    print("Upload successful:", response)
except Exception as e:
    print("Error occurred during upload:", e)