Skip to content

Instantly share code, notes, and snippets.

@vit1-irk

vit1-irk/searching_jsoc.ipynb

Created January 27, 2020 17:37
Show Gist options
  • Save vit1-irk/19d9ebbc69281fd142316f52a2f019ce to your computer and use it in GitHub Desktop.
Save vit1-irk/19d9ebbc69281fd142316f52a2f019ce to your computer and use it in GitHub Desktop.
Download ZIP
Trouble when downloading data from JSOC
Display the source blob
Display the rendered blob
Raw
searching_jsoc.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Broken downloading of magnetic data from JSOC"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import astropy.units as u\n",
"import astropy.time\n",
"import sunpy\n",
"from sunpy.net import jsoc, fido_factory, Fido, attrs as a\n",
"sunpy.log.setLevel(\"DEBUG\")\n",
"import drms"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"==============================\n",
"SunPy Installation Information\n",
"==============================\n",
"\n",
"#######\n",
"General\n",
"#######\n",
"Time : Monday, 27. January 2020 05:36PM UT\n",
"System : Linux\n",
"Processor : \n",
"Arch : 64bit\n",
"SunPy : 1.1.0\n",
"OS: Manjaro Linux 18.1.5 Juhraya (Linux 5.4.14-2-MANJARO )\n",
"\n",
"\n",
"##################\n",
"Required Libraries\n",
"##################\n",
"Python: 3.8.1\n",
"NumPy: 1.18.1\n",
"SciPy: 1.4.1\n",
"matplotlib: 3.1.2\n",
"Astropy: 4.0\n",
"Pandas: 0.25.3\n",
"parfive: 1.0.0\n",
"\n",
"\n",
"#####################\n",
"Recommended Libraries\n",
"#####################\n",
"beautifulsoup: 4.8.2\n",
"PyQt4: NOT INSTALLED\n",
"PyQt5: 5.14.1\n",
"Zeep: 3.4.0\n",
"Sqlalchemy: NOT INSTALLED\n",
"drms: 0.5.7\n"
]
}
],
"source": [
"sunpy.util.system_info()"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['inclination' 'azimuth' 'disambig' 'field' 'vlos_mag' 'dop_width' 'eta_0'\n",
" 'damping' 'src_continuum' 'src_grad' 'alpha_mag' 'chisq' 'conv_flag'\n",
" 'inclination_err' 'azimuth_err' 'field_err' 'vlos_err' 'alpha_err'\n",
" 'field_inclination_err' 'field_az_err' 'inclin_azimuth_err'\n",
" 'field_alpha_err' 'inclination_alpha_err' 'azimuth_alpha_err'\n",
" 'conf_disambig' 'info_map' 'confid_map']\n",
"['T_REC']\n"
]
}
],
"source": [
"series_name = \"hmi.B_720s\"\n",
"notifier = a.jsoc.Notify(\"[email protected]\")\n",
"series = a.jsoc.Series(series_name)\n",
"segments = a.jsoc.Segment('field') & a.jsoc.Segment('inclination') & a.jsoc.Segment('azimuth') & a.jsoc.Segment('disambig')\n",
"\n",
"c = drms.Client()\n",
"si = c.info(series_name).segments.index.values\n",
"print(si)\n",
"print(c.pkeys(series_name))"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Results from 1 Provider:\n",
"\n",
"4 Results from the JSOCClient:\n",
" T_REC TELESCOP INSTRUME WAVELNTH CAR_ROT\n",
" str23 str7 str12 float64 float64\n",
"----------------------- -------- ------------ -------- -------\n",
"2017.09.06_05:36:00_TAI SDO/HMI HMI_COMBINED 6173.0 2194.0\n",
"2017.09.06_05:48:00_TAI SDO/HMI HMI_COMBINED 6173.0 2194.0\n",
"2017.09.06_06:00:00_TAI SDO/HMI HMI_COMBINED 6173.0 2194.0\n",
"2017.09.06_06:12:00_TAI SDO/HMI MISSING 6173.0 --\n",
"\n",
"\n"
]
}
],
"source": [
"attrs_time = a.Time('2017/09/06 05:40', '2017/09/06 06:30')\n",
"res = Fido.search(attrs_time, series, notifier, segments)\n",
"\n",
"print(res)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### The MISSING stuff at the end breaks all the download attempts\n",
"\n",
"It can appear anywhere, not at the end, but also in the middle of the list"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"ename": "DrmsExportError",
"evalue": "Some requested records are offline, try again later: hmi.B_720s[2017.09.06_05:40:37_TAI-2017.09.06_06:30:37_TAI]{field,inclination,azimuth,disambig} [status=4]",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mDrmsExportError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-4-dbd888286a49>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdl_files\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFido\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mres\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprogress\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m/usr/lib/python3.8/site-packages/sunpy/net/fido_factory.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, path, max_conn, progress, overwrite, downloader, *query_results, **kwargs)\u001b[0m\n\u001b[1;32m 379\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mquery_result\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mquery_results\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 380\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mblock\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mquery_result\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresponses\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 381\u001b[0;31m reslist.append(block.client.fetch(block, path=path,\n\u001b[0m\u001b[1;32m 382\u001b[0m \u001b[0mdownloader\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdownloader\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 383\u001b[0m wait=False, **kwargs))\n",
"\u001b[0;32m/usr/lib/python3.8/site-packages/sunpy/net/jsoc/jsoc.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, jsoc_response, path, progress, overwrite, downloader, wait, sleep)\u001b[0m\n\u001b[1;32m 491\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 492\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mresponse\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mresponses\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprogress\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 494\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 495\u001b[0m return self.get_request(responses, path=path, overwrite=overwrite,\n",
"\u001b[0;32m/usr/lib/python3.8/site-packages/drms/client.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout, sleep, retries_notfound, verbose)\u001b[0m\n\u001b[1;32m 470\u001b[0m \u001b[0;31m# We are done, if the request has already finished.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 471\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhas_finished\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mskip_update\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 472\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_raise_on_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 473\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 474\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/lib/python3.8/site-packages/drms/client.py\u001b[0m in \u001b[0;36m_raise_on_error\u001b[0;34m(self, notfound_ok)\u001b[0m\n\u001b[1;32m 210\u001b[0m \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'DRMS export request failed.'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 211\u001b[0m \u001b[0mmsg\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m' [status=%d]'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_status\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 212\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mDrmsExportError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 213\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 214\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_generate_download_urls\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mDrmsExportError\u001b[0m: Some requested records are offline, try again later: hmi.B_720s[2017.09.06_05:40:37_TAI-2017.09.06_06:30:37_TAI]{field,inclination,azimuth,disambig} [status=4]"
]
}
],
"source": [
"dl_files = Fido.fetch(res, progress=False)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<i>Row index=3</i>\n",
"<table id=\"table140144019701136\">\n",
"<thead><tr><th>T_REC</th><th>TELESCOP</th><th>INSTRUME</th><th>WAVELNTH</th><th>CAR_ROT</th></tr></thead>\n",
"<thead><tr><th>str23</th><th>str7</th><th>str12</th><th>float64</th><th>float64</th></tr></thead>\n",
"<tr><td>2017.09.06_06:12:00_TAI</td><td>SDO/HMI</td><td>MISSING</td><td>6173.0</td><td>--</td></tr>\n",
"</table>"
],
"text/plain": [
"<Row index=3>\n",
" T_REC TELESCOP INSTRUME WAVELNTH CAR_ROT\n",
" str23 str7 str12 float64 float64\n",
"----------------------- -------- -------- -------- -------\n",
"2017.09.06_06:12:00_TAI SDO/HMI MISSING 6173.0 --"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"res.get_response(0)[-1]"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.1"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment