HPL.dat calculator - Takes system specs on a single-node basis and calculates N, NB, P, and Q based on available memory and cores

hplcalculator.py

#!/usr/bin/env python3

import math
import psutil

def get_system_info():
    # Get the number of physical cores
    num_cores = psutil.cpu_count(logical=False)

    # Get available physical memory (in bytes)
    available_memory = psutil.virtual_memory().total

    # Convert to GB for easier understanding
    available_memory_gb = available_memory / (1024 ** 3)
    
    return num_cores, available_memory_gb

def calculate_hpl_parameters(available_memory_gb, num_cores, min_N=50000, max_N=1000000, step=50000):
    """
    This function calculates the optimal N, NB, P, and Q for HPL based on available memory and cores.
    :param available_memory_gb: System's available memory in GB
    :param num_cores: Number of physical cores
    :param min_N: Minimum size of N (problem size)
    :param max_N: Maximum size of N
    :param step: Step size for N
    :return: A tuple with (optimal N, NB, P, Q)
    """
    # Assuming HPL will need 8 bytes per double precision element
    double_precision_size = 8  # bytes per double
    memory_per_core_gb = available_memory_gb / num_cores

    best_params = None
    best_performance = float('inf')

    for N in range(min_N, max_N + 1, step):
        # Calculate the optimal NB based on memory per core (rule of thumb)
        # Generally, NB is around sqrt(N) for optimal performance
        NB = int(math.sqrt(N))

        # Try to balance P and Q. The number of processes (P * Q) should be close to the number of cores
        for P in range(1, num_cores + 1):
            for Q in range(1, num_cores + 1):
                if P * Q == num_cores:
                    # Estimate memory requirements for the chosen N, NB, P, and Q
                    required_memory = (N * NB * double_precision_size) / (1024 ** 3)  # in GB
                    if required_memory <= memory_per_core_gb:
                        # Calculate a performance metric here (lower is better)
                        # For now, we assume a simplistic estimation based on memory and grid size
                        performance_metric = required_memory * (P + Q)
                        if performance_metric < best_performance:
                            best_performance = performance_metric
                            best_params = (N, NB, P, Q)

    return best_params

if __name__ == "__main__":
    # Get system information
    num_cores, available_memory_gb = get_system_info()

    print(f"System has {num_cores} physical cores and {available_memory_gb:.2f} GB of memory.")

    # Calculate optimal parameters for HPL
    N, NB, P, Q = calculate_hpl_parameters(available_memory_gb, num_cores)

    print(f"Optimal HPL Parameters:")
    print(f"N: {N}")
    print(f"NB: {NB}")
    print(f"P: {P}")
    print(f"Q: {Q}")

Thanks for this! I'd been tinkering with something automated for my Ansible scripts, but running this and grabbing the output is easier, and doesn't make me feel bad about doing a ton of Python inside YAML :)