lzbench results ran on the 01/04/25 | lzbench 2.0.2 (64-bit Linux) AMD A9-9420 RADEON R5, 5 COMPUTE CORES 2C+3G

calc_optimal.py

import polars as pl
import argparse
import sys
from typing import List, Dict, Tuple

def calculate_optimal_algorithm(
    csv_file: str, 
    file_size_mb: float, 
    upload_speed_mbps: float, 
    download_speed_mbps: float = None,
    visualize: bool = False
):
    """
    Calculate the optimal compression algorithm for a given file size and network speeds.
    
    Args:
        csv_file: Path to the CSV file with compression benchmark data
        file_size_mb: Size of the file in MB
        upload_speed_mbps: Upload speed in Mbps
        download_speed_mbps: Download speed in Mbps (if None, same as upload)
        visualize: Whether to generate visualization plots
    """
    # If download speed not provided, assume same as upload speed
    if download_speed_mbps is None:
        download_speed_mbps = upload_speed_mbps
    
    try:
        df = pl.read_csv(csv_file)
    except Exception as e:
        print(f"Error reading CSV file: {e}")
        sys.exit(1)
    
    if "Compressor name" in df.columns:
        df = df.filter(pl.col("Compressor name") != "memcpy")
    else:
        print("Error: CSV file doesn't have the expected column 'Compressor name'")
        sys.exit(1)
    
    required_columns = ["Compressor name", "Ratio", "Compression speed", "Decompression speed"]
    for col in required_columns:
        if col not in df.columns:
            print(f"Error: Required column '{col}' not found in CSV file")
            sys.exit(1)
    
    df = df.with_columns([
        (file_size_mb * (pl.col("Ratio") / 100) / upload_speed_mbps).alias("upload_time_seconds"),
        (file_size_mb * (pl.col("Ratio") / 100) / download_speed_mbps).alias("download_time_seconds"),
        (file_size_mb / pl.col("Compression speed")).alias("compression_time_seconds"),
        (file_size_mb / pl.col("Decompression speed")).alias("decompression_time_seconds")
    ])
    
    df = df.with_columns([
        (pl.col("compression_time_seconds") + 
         pl.col("upload_time_seconds") + 
         pl.col("download_time_seconds") + 
         pl.col("decompression_time_seconds")).alias("total_time_seconds")
    ])
    
    column_names = df.columns
    
    no_compression_data = {col: [None] for col in column_names}
    
    no_compression_upload_time = file_size_mb / upload_speed_mbps
    no_compression_download_time = file_size_mb / download_speed_mbps
    no_compression_total_time = no_compression_upload_time + no_compression_download_time
    
    no_compression_data.update({
        "Compressor name": ["No compression"],
        "Compression speed": [float('inf')],  
        "Decompression speed": [float('inf')], 
        "Ratio": [100.0],  
        "upload_time_seconds": [no_compression_upload_time],
        "download_time_seconds": [no_compression_download_time],
        "compression_time_seconds": [0.0],  
        "decompression_time_seconds": [0.0],  
        "total_time_seconds": [no_compression_total_time]  
    })
    
    for col in ["Original size", "Compressed size", "Filename"]:
        if col in column_names:
            value = df[col][0] if df.height > 0 else None
            no_compression_data[col] = [value]
    
    no_compression = pl.DataFrame(no_compression_data).select(column_names)
    
    df_with_no_compression = pl.concat([df, no_compression])
    
    df_sorted = df_with_no_compression.sort("total_time_seconds")
    
    top_results = df_sorted.head(15)
    
    print(f"\nOptimal Algorithm Analysis for {file_size_mb:.2f} MB file")
    print(f"Upload speed: {upload_speed_mbps:.2f} Mbps, Download speed: {download_speed_mbps:.2f} Mbps\n")
    
    headers = ["Rank", "Algorithm", "Total (s)", "Comp (s)", "Upload (s)", "Download (s)", "Decomp (s)",
              "Ratio (%)", "Comp Speed", "Decomp Speed"]
    
    header_format = "{:<6} | {:<21} | {:<10} | {:<10} | {:<10} | {:<12} | {:<10} | {:<10} | {:<10} | {:<10}"
    formatted_header = header_format.format(*headers)
    print(formatted_header)
    print("-" * len(formatted_header))
    
    row_format = "{:<6} | {:<21} | {:<10} | {:<10} | {:<10} | {:<12} | {:<10} | {:<10} | {:<10} | {:<10}"
    for i, row in enumerate(top_results.iter_rows(named=True)):
        if row["Compressor name"] == "No compression":
            print(row_format.format(
                i+1,
                row["Compressor name"],
                f"{row['total_time_seconds']:.2f}",
                "N/A",
                f"{row['upload_time_seconds']:.2f}",
                f"{row['download_time_seconds']:.2f}",
                "N/A",
                "100.00",
                "N/A",
                "N/A"
            ))
        else:
            print(row_format.format(
                i+1,
                row["Compressor name"],
                f"{row['total_time_seconds']:.2f}",
                f"{row['compression_time_seconds']:.2f}",
                f"{row['upload_time_seconds']:.2f}",
                f"{row['download_time_seconds']:.2f}",
                f"{row['decompression_time_seconds']:.2f}",
                f"{row['Ratio']:.2f}",
                f"{row['Compression speed']:.2f}",
                f"{row['Decompression speed']:.2f}"
            ))
    
    best_row = top_results.row(0, named=True)
    
    # Find the "No compression" row for comparison
    no_comp_row = None
    for row in df_with_no_compression.iter_rows(named=True):
        if row["Compressor name"] == "No compression":
            no_comp_row = row
            break
    
    if not no_comp_row:
        no_comp_row = {
            "total_time_seconds": no_compression_total_time,
            "upload_time_seconds": no_compression_upload_time,
            "download_time_seconds": no_compression_download_time
        }
    
    print("\n" + "=" * 90)
    print(f"BEST CHOICE: {best_row['Compressor name']}")
    print(f"Estimated total time: {best_row['total_time_seconds']:.2f} seconds")
    
    time_savings = no_comp_row["total_time_seconds"] - best_row["total_time_seconds"]
    time_savings_pct = (time_savings / no_comp_row["total_time_seconds"]) * 100
    
    if best_row['Compressor name'] == 'No compression':
        print(f"Time breakdown: 0.00s compression + {best_row['upload_time_seconds']:.2f}s upload + "
              f"{best_row['download_time_seconds']:.2f}s download + 0.00s decompression")
        print("RECOMMENDATION: Don't compress the file, just transfer it directly!")
    else:
        print(f"Time breakdown: {best_row['compression_time_seconds']:.2f}s compression + "
              f"{best_row['upload_time_seconds']:.2f}s upload + "
              f"{best_row['download_time_seconds']:.2f}s download + "
              f"{best_row['decompression_time_seconds']:.2f}s decompression")
        
        compressed_size = file_size_mb * (best_row['Ratio'] / 100)
        savings_pct = 100 - best_row['Ratio']
        print(f"File size reduction: {file_size_mb:.2f} MB → {compressed_size:.2f} MB ({savings_pct:.1f}% savings)")
        
        if time_savings > 0:
            print(f"Time savings vs no compression: {time_savings:.2f} seconds ({time_savings_pct:.1f}% faster)")
            hours_saved = time_savings // 3600
            minutes_saved = (time_savings % 3600) // 60
            seconds_saved = time_savings % 60
            
            if hours_saved > 0:
                print(f"That's {int(hours_saved)}h {int(minutes_saved)}m {int(seconds_saved)}s saved!")
            elif minutes_saved > 0:
                print(f"That's {int(minutes_saved)}m {int(seconds_saved)}s saved!")
        else:
            print(f"NOTE: Compression adds {-time_savings:.2f} seconds compared to direct transfer.")
            print("Compression is not beneficial for this combination of file size and network speed.")
    
    print("=" * 90)
    
    if best_row['Compressor name'] != 'No compression':
        print("\nINSIGHTS:")
        times = {
            'Compression': best_row['compression_time_seconds'],
            'Upload': best_row['upload_time_seconds'],
            'Download': best_row['download_time_seconds'],
            'Decompression': best_row['decompression_time_seconds']
        }
        bottleneck = max(times, key=times.get)
        bottleneck_pct = (times[bottleneck] / best_row['total_time_seconds']) * 100
        
        print(f"- {bottleneck} is the bottleneck in your scenario ({times[bottleneck]:.2f}s, {bottleneck_pct:.1f}% of total time)")
        
        if bottleneck == 'Compression':
            faster_comp = df.filter(
                (pl.col("Compression speed") > best_row['Compression speed'] * 1.3) & 
                (pl.col("Ratio") < best_row['Ratio'] * 1.1) 
            )
            if faster_comp.height > 0:
                faster_comp = faster_comp.sort("Compression speed", descending=True)
                alt = faster_comp.row(0, named=True)
                speedup = alt['Compression speed'] / best_row['Compression speed']
                print(f"- Consider: {alt['Compressor name']} - {speedup:.1f}x faster compression with only slightly worse ratio")
                print(f"  Potential time savings: {best_row['compression_time_seconds'] - (file_size_mb / alt['Compression speed']):.2f}s")
        
        elif bottleneck == 'Decompression':
            faster_decomp = df.filter(
                (pl.col("Decompression speed") > best_row['Decompression speed'] * 1.3) & 
                (pl.col("Ratio") < best_row['Ratio'] * 1.1)  
            )
            if faster_decomp.height > 0:
                faster_decomp = faster_decomp.sort("Decompression speed", descending=True)
                alt = faster_decomp.row(0, named=True)
                speedup = alt['Decompression speed'] / best_row['Decompression speed']
                print(f"- Consider: {alt['Compressor name']} - {speedup:.1f}x faster decompression with only slightly worse ratio")
                print(f"  Potential time savings: {best_row['decompression_time_seconds'] - (file_size_mb / alt['Decompression speed']):.2f}s")
        
        elif bottleneck in ['Upload', 'Download']:
            better_ratio = df.filter(
                (pl.col("Ratio") < best_row['Ratio'] * 0.9) & 
                (pl.col("Compression speed") > best_row['Compression speed'] * 0.5) &
                (pl.col("Decompression speed") > best_row['Decompression speed'] * 0.5)
            )
            if better_ratio.height > 0:
                better_ratio = better_ratio.sort("Ratio")
                alt = better_ratio.row(0, named=True)
                ratio_improvement = (best_row['Ratio'] - alt['Ratio']) / best_row['Ratio'] * 100
                
                current_transfer = best_row['upload_time_seconds'] + best_row['download_time_seconds']
                alt_transfer = (file_size_mb * (alt['Ratio'] / 100) / upload_speed_mbps) + (file_size_mb * (alt['Ratio'] / 100) / download_speed_mbps)
                transfer_savings = current_transfer - alt_transfer
                
                current_processing = best_row['compression_time_seconds'] + best_row['decompression_time_seconds']
                alt_processing = (file_size_mb / alt['Compression speed']) + (file_size_mb / alt['Decompression speed'])
                processing_penalty = alt_processing - current_processing
                
                net_savings = transfer_savings - processing_penalty
                
                if net_savings > 0:
                    print(f"- Consider: {alt['Compressor name']} - {ratio_improvement:.1f}% better compression ratio")
                    print(f"  Potential net time savings: {net_savings:.2f}s (Transfer: -{transfer_savings:.2f}s, Processing: +{processing_penalty:.2f}s)")
    
    print("\nALGORITHM RECOMMENDATIONS BY NETWORK SPEED:")
    
    test_speeds = [0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000]
    algorithms_by_speed = {}
    
    for speed in test_speeds:
        test_df = df.with_columns([
            (file_size_mb * (pl.col("Ratio") / 100) / speed).alias("test_upload_time"),
            (file_size_mb * (pl.col("Ratio") / 100) / speed).alias("test_download_time"),
            (pl.col("compression_time_seconds") + 
             (file_size_mb * (pl.col("Ratio") / 100) / speed) +
             (file_size_mb * (pl.col("Ratio") / 100) / speed) +
             pl.col("decompression_time_seconds")).alias("test_total_time")
        ])
        
        no_comp_time = file_size_mb / speed * 2  # upload + download
        
        test_df = test_df.sort("test_total_time")
        
        if test_df.height > 0:
            best_algo = test_df.row(0, named=True)
            # Check if compression is actually beneficial at this speed
            if best_algo["test_total_time"] < no_comp_time:
                algorithms_by_speed[speed] = best_algo["Compressor name"]
            else:
                algorithms_by_speed[speed] = "No compression"
        else:
            algorithms_by_speed[speed] = "Unknown"
    
    algo_to_speeds = {}
    for speed, algo in algorithms_by_speed.items():
        if algo not in algo_to_speeds:
            algo_to_speeds[algo] = []
        algo_to_speeds[algo].append(speed)
    
    for algo, speeds in algo_to_speeds.items():
        if len(speeds) > 0:
            min_speed = min(speeds)
            max_speed = max(speeds)
            
            if min_speed == max_speed:
                print(f"- {algo}: Optimal at {min_speed} Mbps")
            else:
                if speeds == test_speeds:
                    print(f"- {algo}: Optimal across all tested network speeds ({min_speed}-{max_speed} Mbps)")
                elif min_speed == test_speeds[0]:
                    print(f"- {algo}: Optimal for slow networks up to {max_speed} Mbps")
                elif max_speed == test_speeds[-1]:
                    print(f"- {algo}: Optimal for fast networks {min_speed}+ Mbps")
                else:
                    print(f"- {algo}: Optimal for networks between {min_speed}-{max_speed} Mbps")
    
    if visualize:
        try:
            import matplotlib.pyplot as plt
            
            top_5_algos = list(top_results.filter(pl.col("Compressor name") != "No compression").head(5)["Compressor name"])
            top_5_algos.append("No compression")
            
            # Test more granular speeds for smoother curves
            vis_speeds = sorted(list(set([x/10 for x in range(1, 101)] + [x for x in range(10, 1001, 10)])))
            
            # For each algorithm, calculate times at different network speeds
            algo_times = {algo: [] for algo in top_5_algos}
            no_comp_times = []
            
            for speed in vis_speeds:
                no_comp_time = file_size_mb / speed * 2  # Upload + download
                no_comp_times.append(no_comp_time)
                
                for algo in top_5_algos:
                    if algo == "No compression":
                        continue
                    
                    algo_data = df.filter(pl.col("Compressor name") == algo)
                    if algo_data.height > 0:
                        row = algo_data.row(0, named=True)
                        comp_time = row["compression_time_seconds"]
                        decomp_time = row["decompression_time_seconds"]
                        transfer_time = file_size_mb * (row["Ratio"] / 100) / speed * 2  # Upload + download
                        total_time = comp_time + transfer_time + decomp_time
                        algo_times[algo].append(total_time)
                    else:
                        algo_times[algo].append(float('inf'))
            
            algo_times["No compression"] = no_comp_times
            
            plt.figure(figsize=(12, 8))
            
            for algo in top_5_algos:
                plt.plot(vis_speeds, algo_times[algo], label=algo)
            
            plt.xscale('log')  # Log scale for network speed
            plt.yscale('log')  # Log scale for time
            
            plt.xlabel('Network Speed (Mbps)')
            plt.ylabel('Total Transfer Time (seconds)')
            plt.title(f'Compression Algorithm Performance vs Network Speed for {file_size_mb} MB File')
            plt.grid(True, which="both", ls="--")
            plt.legend()
            
            plt.axvline(x=upload_speed_mbps, color='r', linestyle='--', alpha=0.7)
            plt.text(upload_speed_mbps*1.1, plt.ylim()[0]*1.1, f'Current: {upload_speed_mbps} Mbps', 
                     rotation=90, verticalalignment='bottom')
            
            plt.tight_layout()
            plt.savefig('compression_performance.png')
            print("\nVisualization saved as 'compression_performance.png'")
            
        except ImportError:
            print("\nVisualization skipped: matplotlib not available")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Calculate optimal compression algorithm based on file size and network speeds')
    parser.add_argument('file_size', type=float, help='File size in MB')
    parser.add_argument('upload_speed', type=float, help='Upload speed in Mbps (megabits per second)')
    parser.add_argument('--download_speed', type=float, help='Download speed in Mbps (if different from upload)')
    parser.add_argument('--csv', default='compression_benchmark.csv', help='Path to compression benchmark CSV file')
    parser.add_argument('--visualize', action='store_true', help='Generate visualization (requires matplotlib)')
    
    args = parser.parse_args()
    
    calculate_optimal_algorithm(args.csv, args.file_size, args.upload_speed, args.download_speed, args.visualize)

compression_performance.png

output example

$ python3 calc2.py --csv results.csv 100 10 --visualize

Optimal Algorithm Analysis for 100.00 MB file
Upload speed: 10.00 Mbps, Download speed: 10.00 Mbps

Rank   | Algorithm             | Total (s)  | Comp (s)   | Upload (s) | Download (s) | Decomp (s) | Ratio (%)  | Comp Speed | Decomp Speed
------------------------------------------------------------------------------------------------------------------------------------------
1      | zstd 1.5.7 -2         | 7.35       | 0.67       | 3.27       | 3.27         | 0.14       | 32.70      | 148.41     | 734.87    
2      | zstd 1.5.7 -1         | 7.40       | 0.38       | 3.45       | 3.45         | 0.12       | 34.53      | 263.33     | 863.57    
3      | libdeflate 1.23 -1    | 7.74       | 0.64       | 3.47       | 3.47         | 0.17       | 34.68      | 155.98     | 601.58    
4      | libdeflate 1.23 -3    | 7.92       | 1.13       | 3.31       | 3.31         | 0.17       | 33.11      | 88.59      | 602.23    
5      | libdeflate 1.23 -6    | 8.21       | 1.68       | 3.19       | 3.19         | 0.16       | 31.85      | 59.56      | 618.05    
6      | lizard 2.1 -40        | 8.30       | 0.55       | 3.81       | 3.81         | 0.12       | 38.14      | 182.77     | 825.37    
7      | zstd 1.5.7 -5         | 8.32       | 2.22       | 2.96       | 2.96         | 0.18       | 29.60      | 44.99      | 557.39    
8      | lizard 2.1 -32        | 8.42       | 0.91       | 3.71       | 3.71         | 0.09       | 37.11      | 109.59     | 1130.96   
9      | brotli 1.1.0 -0       | 8.44       | 0.64       | 3.70       | 3.70         | 0.39       | 37.02      | 155.21     | 253.38    
10     | brotli 1.1.0 -2       | 8.45       | 1.67       | 3.21       | 3.21         | 0.35       | 32.12      | 59.99      | 281.77    
11     | kanzi 2.3 -2          | 8.51       | 1.75       | 3.22       | 3.22         | 0.32       | 32.22      | 57.13      | 315.09    
12     | lizard 2.1 -30        | 8.56       | 0.37       | 4.05       | 4.05         | 0.10       | 40.49      | 270.83     | 1039.30   
13     | zstd 1.5.7 --fast --1 | 8.65       | 0.35       | 4.10       | 4.10         | 0.10       | 41.01      | 288.62     | 1031.99   
14     | zlib-ng 2.2.3 -6      | 8.81       | 2.08       | 3.25       | 3.25         | 0.23       | 32.49      | 48.06      | 436.84    
15     | quicklz 1.5.0 -2      | 8.89       | 0.64       | 3.99       | 3.99         | 0.27       | 39.89      | 155.06     | 371.67    

==========================================================================================
BEST CHOICE: zstd 1.5.7 -2
Estimated total time: 7.35 seconds
Time breakdown: 0.67s compression + 3.27s upload + 3.27s download + 0.14s decompression
File size reduction: 100.00 MB → 32.70 MB (67.3% savings)
Time savings vs no compression: 12.65 seconds (63.3% faster)
==========================================================================================

INSIGHTS:
- Upload is the bottleneck in your scenario (3.27s, 44.5% of total time)

ALGORITHM RECOMMENDATIONS BY NETWORK SPEED:
- bsc 3.3.5 -m0 -e1: Optimal at 0.5 Mbps
- zstd 1.5.7 -8: Optimal at 1 Mbps
- zstd 1.5.7 -5: Optimal at 2 Mbps
- zstd 1.5.7 -2: Optimal for networks between 5-10 Mbps
- zstd 1.5.7 -1: Optimal for networks between 20-100 Mbps
- lz4 1.10.0 --fast -3: Optimal at 200 Mbps
- No compression: Optimal for fast networks 500+ Mbps

Visualization saved as 'compression_performance.png'

results.csv