Karabo data cache format experimentation

cache_format_options.py

import json
import h5py
import numpy as np

def read_json(path):
    with open(path) as f:
        loaded_data = json.load(f)

    res = {}
    for info in loaded_data:
        fname = info['filename']
        res[fname] = {
            'filename': fname,
            'mtime': info['mtime'],
            'size': info['size'],
            'train_ids': np.array(info['train_ids'], dtype=np.uint64),
            'control_sources': frozenset(info['control_sources']),
            'instrument_sources': frozenset(info['instrument_sources']),
        }

    return res

str_dtype = h5py.special_dtype(vlen=str)

def save_hdf5(data_by_file, path):
    with h5py.File(path, 'w') as f:
        for fname, info in data_by_file.items():
            fname_grp = f.create_group(fname)
            fname_grp.attrs['mtime'] = info['mtime']
            fname_grp.attrs['size'] = info['size']
            fname_grp['train_ids'] = info['train_ids']
            cs = fname_grp.create_dataset(
                'control_sources',
                shape=(len(info['control_sources']),),
                dtype=str_dtype,
            )
            cs[:] = sorted(info['control_sources'])
            is_ = fname_grp.create_dataset(
                'instrument_sources',
                shape=(len(info['instrument_sources']),),
                dtype=str_dtype,
            )
            is_[:] = sorted(info['instrument_sources'])

def read_hdf5(path, core=False):
    driver_kwargs = {}
    if core:
        driver_kwargs = {'driver': 'core', 'backing_store': False}

    res = {}
    with h5py.File(path, 'r', **driver_kwargs) as f:
        for fname, group in f.items():
            res[fname] = {
                'filename': fname,
                'mtime': group.attrs['mtime'],
                'size': group.attrs['size'],
                'train_ids': group['train_ids'][:],
                'control_sources': frozenset(group['control_sources'][:]),
                'instrument_sources': frozenset(group['instrument_sources'][:]),
            }

    return res