Skip to content

Instantly share code, notes, and snippets.

@mducle

mducle/lucan_materialize.ipynb

Created October 9, 2024 16:58
Show Gist options
  • Save mducle/a24d09afe8365ee5c3a6c5469f540904 to your computer and use it in GitHub Desktop.
Save mducle/a24d09afe8365ee5c3a6c5469f540904 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
lucan_materialize.ipynb
{
"cells": [
{
"cell_type": "markdown",
"id": "31bff45f-84a4-47dd-b957-b014c02945bb",
"metadata": {},
"source": [
"Notebook tested on Windows 10, in a conda environment setup with:\n",
"\n",
"```\n",
"mamba create -n lucan python=3.10 vaex jupyter\n",
"mamba activate lucan\n",
"python -m pip install pace_neutrons\n",
"cd d:/src/lucan\n",
"python -m pip install -e .\n",
"jupyter notebook\n",
"```\n",
"\n",
"Data files can be download from [this Zenodo archive](https://zenodo.org/records/5020485)"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "908fc6ac-4b1f-4381-9c8c-09c6357f8320",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Config not found, using default settings. To preserve settings between sessions run `lucan.settings.settings.save()`.\n"
]
}
],
"source": [
"from pathlib import Path\n",
"\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"\n",
"import lucan\n",
"from lucan.constants import Emode\n",
"from lucan.gen_sqw import gen_sqw_from_nxspe\n",
"from lucan.goniometers import Goniometer\n",
"from lucan.samples import Sample\n",
"from lucan.sqw import SQw\n",
"from lucan.projections import INSTRUMENT_PROJECTION, LinearProjection\n",
"\n",
"# Set up\n",
"h5path = Path(\"d:/src/edatc/data/\")\n",
"nxspes = [Path(h5path / f\"map{i}_ei400.nxspe\") for i in range(15052, 15098)]\n",
"out_file = Path(\"d:/src/lucan/output.h5\")\n",
"\n",
"gon = Goniometer.from_zip(u=[1, 0, 0], v=[0, 1, 0],\n",
" psi=range(0,91,2), omega=0., dpsi=0., gl=0., gs=0.,\n",
" degrees=True)\n",
"sample = Sample(\"geoff\", [2.87, 2.87, 2.87], [90.0, 90.0, 90.0])"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "609072b7-e892-4243-b428-5570e3634d57",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h \n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.01s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 1.85s = 0.0m = 0.0h \n",
"sum [########################################] 100.00% elapsed time : 1.84s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 1.83s = 0.0m = 0.0h\n",
" "
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"D:\\src\\lucan\\src\\lucan\\sqw.py:418: UserWarning: rename 'ren' to 'E' does not create an index anymore. Try using swap_dims instead or use set_index after rename to create an indexed coordinate.\n",
" .rename(dict(zip(tmp_coords, proj.INDICES)))\n"
]
}
],
"source": [
"x = SQw.load(out_file)\n",
"# Make a cut over all the data\n",
"j = x.cut(LinearProjection([-1, 1, 0], [1, 1, 0]), [(-13.025, 13.025, 0.05), (-13.014, 13.014, 0.03), (-100, 100), (-100, 1000)])"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "ca91f00c-3799-4b27-a776-e83f12307356",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:19<00:00, 9.77s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"19.5 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n"
]
}
],
"source": [
"%%timeit -n1 -r1\n",
"rinsp = j.inspect_runs([1,2])"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "2f3f78e8-5101-4108-a3e6-edd5d696fff4",
"metadata": {},
"outputs": [],
"source": [
"j._data.materialize(inplace=True);"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "de97bfd7-cc9a-47a2-8692-25a5def3a30d",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:17<00:00, 9.00s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"18 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n"
]
}
],
"source": [
"%%timeit -n1 -r1\n",
"rinsp = j.inspect_runs([1,2])"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "ce8c61f9-ec4b-4dd3-bad7-5c8b95224914",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.01s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h \n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 1.68s = 0.0m = 0.0h \n",
"sum [########################################] 100.00% elapsed time : 1.67s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 1.66s = 0.0m = 0.0h\n",
" "
]
}
],
"source": [
"import vaex\n",
"x = SQw.load(out_file)\n",
"# Make a cut over all the data\n",
"j = x.cut(LinearProjection([-1, 1, 0], [1, 1, 0]), [(-13.025, 13.025, 0.05), (-13.014, 13.014, 0.03), (-100, 100), (-100, 1000)])\n",
"j._data = vaex.from_pandas(j.data.to_pandas_df())"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "47a60711-1ff1-4a6a-9761-bd6b0599ed25",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:19<00:00, 9.86s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"19.7 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"%%timeit -n1 -r1\n",
"rinsp = j.inspect_runs([1,2])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4c2f73e0-1cf1-4401-86b7-39960c019000",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 1,
"id": "7ff4e586-4643-4fed-9765-e95c4f08815d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Config not found, using default settings. To preserve settings between sessions run `lucan.settings.settings.save()`.\n"
]
}
],
"source": [
"from pathlib import Path\n",
"\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"\n",
"import lucan\n",
"from lucan.constants import Emode\n",
"from lucan.gen_sqw import gen_sqw_from_nxspe\n",
"from lucan.goniometers import Goniometer\n",
"from lucan.samples import Sample\n",
"from lucan.sqw import SQw\n",
"from lucan.projections import INSTRUMENT_PROJECTION, LinearProjection\n",
"\n",
"# Set up\n",
"h5path = Path(\"d:/src/edatc/data/\")\n",
"nxspes = [Path(h5path / f\"map{i}_ei400.nxspe\") for i in range(15052, 15098)]\n",
"out_file = Path(\"d:/src/lucan/output.h5\")\n",
"\n",
"gon = Goniometer.from_zip(u=[1, 0, 0], v=[0, 1, 0],\n",
" psi=range(0,91,2), omega=0., dpsi=0., gl=0., gs=0.,\n",
" degrees=True)\n",
"sample = Sample(\"geoff\", [2.87, 2.87, 2.87], [90.0, 90.0, 90.0])"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "3eaf5314-a4a5-45ad-9918-01fbd5c9902c",
"metadata": {},
"outputs": [],
"source": [
"x = SQw.load(out_file)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "e5dd8b90-4d1d-4938-8d8e-56c8956bc0cc",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h \n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 1.77s = 0.0m = 0.0h \n",
"sum [########################################] 100.00% elapsed time : 1.76s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 1.75s = 0.0m = 0.0h\n",
" "
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"D:\\src\\lucan\\src\\lucan\\sqw.py:418: UserWarning: rename 'ren' to 'E' does not create an index anymore. Try using swap_dims instead or use set_index after rename to create an indexed coordinate.\n",
" .rename(dict(zip(tmp_coords, proj.INDICES)))\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"9.41 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n"
]
}
],
"source": [
"%%timeit -n1 -r1\n",
"j = x.cut(LinearProjection([-1, 1, 0], [1, 1, 0]), [(-3.025, 3.025, 0.05), (-3.014, 3.014, 0.03), (-0.1, 0.1), (75., 85.)])"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "04d5dd71-bdb2-4845-9a2f-7d78d5c45993",
"metadata": {},
"outputs": [],
"source": [
"x._data.materialize(inplace=True);"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "dbe3e5f6-1114-4096-ab45-1492a7e5697c",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.01s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h \n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.01s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 1.52s = 0.0m = 0.0h \n",
"sum [########################################] 100.00% elapsed time : 1.50s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 1.49s = 0.0m = 0.0h\n",
" 8.64 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n"
]
}
],
"source": [
"%%timeit -n1 -r1\n",
"j = x.cut(LinearProjection([-1, 1, 0], [1, 1, 0]), [(-3.025, 3.025, 0.05), (-3.014, 3.014, 0.03), (-0.1, 0.1), (75., 85.)])"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "6a0f45b8-b040-484f-b723-d88b9eda4703",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.01s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h \n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 1.52s = 0.0m = 0.0h \n",
"sum [########################################] 100.00% elapsed time : 1.50s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 1.49s = 0.0m = 0.0h\n",
" 8.66 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n"
]
}
],
"source": [
"%%timeit -n1 -r1\n",
"j = x.cut(LinearProjection([-1, 1, 0], [1, 1, 0]), [(-3.025, 3.025, 0.05), (-3.014, 3.014, 0.03), (-0.1, 0.1), (75., 85.)])"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "8207a0eb-41b8-46fd-8ecc-7ff281f653b7",
"metadata": {},
"outputs": [],
"source": [
"import vaex\n",
"x = SQw.load(out_file)\n",
"x._data = vaex.from_pandas(x.data.to_pandas_df())"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "a8649e26-85b9-4180-ab42-ce664ad79194",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.01s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h \n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.01s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 1.56s = 0.0m = 0.0h \n",
"sum [########################################] 100.00% elapsed time : 1.55s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 1.54s = 0.0m = 0.0h\n",
" 10.4 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n"
]
}
],
"source": [
"%%timeit -n1 -r1\n",
"j = x.cut(LinearProjection([-1, 1, 0], [1, 1, 0]), [(-3.025, 3.025, 0.05), (-3.014, 3.014, 0.03), (-0.1, 0.1), (75., 85.)])"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "31a045a1-6648-4365-b168-f8b515337430",
"metadata": {},
"outputs": [],
"source": [
"x = SQw.load(out_file)\n",
"x._data.materialize(inplace=True)\n",
"# Evaluate expressions on all columns to load them into memory\n",
"for col in x._data.get_column_names(hidden=True):\n",
" x._data[col].sum()"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "a7f40f4a-32fe-416f-bbf1-1c92842586e0",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 0.01s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h \n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"sum [########################################] 100.00% elapsed time : 0.01s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.00s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 0.01s = 0.0m = 0.0h\n",
"mean [########################################] 100.00% elapsed time : 1.59s = 0.0m = 0.0h \n",
"sum [########################################] 100.00% elapsed time : 1.58s = 0.0m = 0.0h\n",
"count [########################################] 100.00% elapsed time : 1.56s = 0.0m = 0.0h\n",
" 9.4 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n"
]
}
],
"source": [
"%%timeit -n1 -r1\n",
"j = x.cut(LinearProjection([-1, 1, 0], [1, 1, 0]), [(-3.025, 3.025, 0.05), (-3.014, 3.014, 0.03), (-0.1, 0.1), (75., 85.)])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6eea5475-ee56-45b6-9c10-8fb1f517517b",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.15"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment