diff --git a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
index 134e7e0bb729e7faf6c1509ea6d65c46156709d2..b455ca34d83b62c720521b93b351e917df945f6a 100644
--- a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
+++ b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
@@ -112,7 +112,6 @@
"if not karabo_id_control:\n",
" karabo_id_control = karabo_id\n",
"\n",
- "instrument_src = instrument_source_template.format(karabo_id, receiver_template)\n",
"ctrl_src = ctrl_source_template.format(karabo_id_control, control_template)\n",
"\n",
"print(f\"Process modules: {karabo_da} for run {run}\")\n",
@@ -160,6 +159,22 @@
"step_timer = StepTimer()"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7bf7fb32",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_dc = RunDirectory(run_folder)\n",
+ "\n",
+ "# Decide if we are processing 2 25um or 1 50um Gotthard-II\n",
+ "\n",
+ "receivers = list(run_dc.select(f'{karabo_id}/DET/{receiver_template}*').all_sources)\n",
+ "\n",
+ "corr_karabo_da = receivers[0].split(\"/\")[-1].split(\":\")[0][:-2]"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -168,7 +183,6 @@
"outputs": [],
"source": [
"# Read slow data\n",
- "run_dc = RunDirectory(run_folder)\n",
"g2ctrl = gotthard2lib.Gotthard2Ctrl(run_dc=run_dc, ctrl_src=ctrl_src)\n",
"\n",
"if bias_voltage == -1:\n",
@@ -186,7 +200,13 @@
"print(\"Exposure Time:\", exposure_time)\n",
"print(\"Exposure Period:\", exposure_period)\n",
"print(\"Acquisition Rate:\", acquisition_rate)\n",
- "print(\"Single Photon:\", single_photon)"
+ "print(\"Single Photon:\", single_photon)\n",
+ "\n",
+ "# Decide if GH2 is 25um or 50um\n",
+ "gh2_hostname = run_dc.get_run_value(ctrl_src, \"rxHostname\")\n",
+ "# gh2_hostname is a vector of bytes objects.\n",
+ "# GH2 25um has two hostnames unlike 50um which has one.\n",
+ "gh2_detector = \"25um\" if gh2_hostname[1].decode(\"utf-8\") else \"50um\""
]
},
{
@@ -229,7 +249,7 @@
" db_modules = [None] * len(karabo_da)\n",
"\n",
"if constants_file:\n",
- " for mod in karabo_da:\n",
+ " for mod in receivers:\n",
" const_data[mod] = dict()\n",
" # load constants temporarily using defined local paths.\n",
" with h5py.File(constants_file, \"r\") as cfile:\n",
@@ -327,9 +347,9 @@
" data_corr[index, ...],\n",
" mask[index, ...],\n",
" g,\n",
- " const_data[mod][\"OffsetGotthard2\"],\n",
- " const_data[mod][\"RelativeGainGotthard2\"], \n",
- " const_data[mod][\"Mask\"],\n",
+ " const_data[rec_mod][\"OffsetGotthard2\"],\n",
+ " const_data[rec_mod][\"RelativeGainGotthard2\"], \n",
+ " const_data[rec_mod][\"Mask\"],\n",
" apply_offset=offset_correction,\n",
" apply_gain=gain_correction,\n",
" )"
@@ -342,22 +362,29 @@
"metadata": {},
"outputs": [],
"source": [
- "for mod in karabo_da:\n",
- " # This is used in case receiver template consists of\n",
- " # karabo data aggregator index. e.g. detector at DETLAB\n",
- " instr_mod_src = instrument_src.format(mod[-2:])\n",
- " data_path = \"INSTRUMENT/\" + instr_mod_src + \"/data\"\n",
- " for raw_file in seq_files:\n",
- " step_timer.start()\n",
+ "for raw_file in seq_files:\n",
"\n",
- " dc = H5File(raw_file)\n",
- " out_file = out_folder / raw_file.name.replace(\"RAW\", \"CORR\")\n",
+ " out_file = out_folder / raw_file.name.replace(\"RAW\", \"CORR\")\n",
+ " # Select module INSTRUMENT sources and deselect empty trains.\n",
+ " dc = H5File(raw_file).select(receivers, require_all=True)\n",
"\n",
- " # Select module INSTRUMENT source and deselect empty trains.\n",
- " dc = dc.select(instr_mod_src, require_all=True)\n",
- " data = dc[instr_mod_src, \"data.adc\"].ndarray()\n",
- " gain = dc[instr_mod_src, \"data.gain\"].ndarray()\n",
- " step_timer.done_step(\"preparing raw data\")\n",
+ " n_trains = len(dc.train_ids)\n",
+ "\n",
+ " # Initialize GH2 data and gain arrays to store in corrected files.\n",
+ " if gh2_detector == \"25um\":\n",
+ " data_stored = np.zeros((dc[receivers[0], \"data.adc\"].shape[:2] + (1280 * 2,)), dtype=np.float32)\n",
+ " gain_stored = np.zeros((dc[receivers[0], \"data.adc\"].shape[:2] + (1280 * 2,)), dtype=np.uint8)\n",
+ " else:\n",
+ " data_stored = None\n",
+ " gain_stored = None\n",
+ "\n",
+ " for i, rec_mod in enumerate(receivers):\n",
+ " step_timer.start()\n",
+ " print(f\"Correcting {rec_mod} for {raw_file}\")\n",
+ "\n",
+ " data = dc[rec_mod, \"data.adc\"].ndarray()\n",
+ " gain = dc[rec_mod, \"data.gain\"].ndarray()\n",
+ " step_timer.done_step(\"Preparing raw data\")\n",
" dshape = data.shape\n",
"\n",
" step_timer.start()\n",
@@ -366,7 +393,7 @@
" data_corr = context.alloc(shape=dshape, dtype=np.float32)\n",
" mask = context.alloc(shape=dshape, dtype=np.uint32)\n",
" context.map(correct_train, data)\n",
- " step_timer.done_step(\"Correcting one sequence file\")\n",
+ " step_timer.done_step(f\"Correcting one receiver in one sequence file\")\n",
"\n",
" step_timer.start()\n",
"\n",
@@ -375,48 +402,56 @@
" data_corr[np.isinf(data_corr)] = np.nan\n",
"\n",
" # Create CORR files and add corrected data sections.\n",
- " image_counts = dc[instrument_src, \"data.adc\"].data_counts(labelled=False)\n",
- "\n",
- " with DataFile(out_file, \"w\") as ofile:\n",
- " # Create INDEX datasets.\n",
- " ofile.create_index(dc.train_ids, from_file=dc.files[0])\n",
- " # Create METDATA datasets\n",
- " ofile.create_metadata(\n",
- " like=dc,\n",
- " sequence=dc.run_metadata()[\"sequenceNumber\"],\n",
- " instrument_channels=(f\"{instrument_src}/data\",)\n",
- " )\n",
+ " image_counts = dc[rec_mod, \"data.adc\"].data_counts(labelled=False)\n",
+ "\n",
+ " if gh2_detector == \"25um\":\n",
+ " data_stored[..., i::2] = data_corr.copy()\n",
+ " gain_stored[..., i::2] = gain.copy()\n",
+ " else: # \"50um\"\n",
+ " data_stored = data_corr\n",
+ " gain_stored = gain\n",
+ "\n",
+ " with DataFile(out_file, \"w\") as ofile:\n",
+ " # Create INDEX datasets.\n",
+ " ofile.create_index(dc.train_ids, from_file=dc.files[0])\n",
+ " ofile.create_metadata(\n",
+ " like=dc,\n",
+ " sequence=dc.run_metadata()[\"sequenceNumber\"],\n",
+ " instrument_channels=(f\"{instrument_src}/data\",)\n",
+ " )\n",
"\n",
- " # Create Instrument section to later add corrected datasets.\n",
- " outp_source = ofile.create_instrument_source(instrument_src)\n",
+ " # Create Instrument section to later add corrected datasets.\n",
+ " outp_source = ofile.create_instrument_source(f\"{karabo_id}/DET/{corr_karabo_da}:daqOutput\")\n",
"\n",
- " # Create count/first datasets at INDEX source.\n",
- " outp_source.create_index(data=image_counts)\n",
+ " # Create count/first datasets at INDEX source.\n",
+ " outp_source.create_index(data=image_counts)\n",
"\n",
- " # Store uncorrected trainId in the corrected file.\n",
- " outp_source.create_key(\n",
- " f\"data.trainId\", data=dc.train_ids,\n",
- " chunks=min(50, len(dc.train_ids))\n",
- " )\n",
- "\n",
- " # Create datasets with the available corrected data\n",
- " for field_name, field_data in {\n",
- " \"adc\": data_corr,\n",
- " \"gain\": gain,\n",
- " }.items():\n",
- " outp_source.create_key(\n",
- " f\"data.{field_name}\", data=field_data,\n",
- " chunks=((chunks_data,) + data_corr.shape[1:])\n",
+ " # Store uncorrected trainId in the corrected file.\n",
+ " outp_source.create_key(\n",
+ " f\"data.trainId\", data=dc.train_ids,\n",
+ " chunks=min(50, len(dc.train_ids))\n",
" )\n",
"\n",
- " for field in [\"bunchId\", \"memoryCell\", \"frameNumber\", \"timestamp\"]:\n",
+ " # Create datasets with the available corrected data\n",
+ " for field_name, field_data in {\n",
+ " \"adc\": data_stored,\n",
+ " \"gain\": gain_stored,\n",
+ " }.items():\n",
+ " outp_source.create_key(\n",
+ " f\"data.{field_name}\", data=field_data,\n",
+ " chunks=((chunk_size_idim,) + data_corr.shape[1:])\n",
+ " )\n",
+ "\n",
+ " # For GH2 25um, the data of the second receiver is\n",
+ " # stored in the corrected file.\n",
+ " for field in [\"bunchId\", \"memoryCell\", \"frameNumber\", \"timestamp\"]:\n",
" outp_source.create_key(\n",
" f\"data.{field}\", data=dc[instr_mod_src, f\"data.{field}\"].ndarray(),\n",
" chunks=(chunks_data, data_corr.shape[1])\n",
" )\n",
- " outp_source.create_compressed_key(f\"data.mask\", data=mask)\n",
- "\n",
- " step_timer.done_step(\"Storing data\")"
+ " outp_source.create_compressed_key(f\"data.mask\", data=mask)\n",
+ " step_timer.done_step(\"Storing data\")\n",
+ "\n"
]
},
{
@@ -430,6 +465,16 @@
"step_timer.print_summary()"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7f366352",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "corr_data_source = f\"{karabo_id}/DET/{corr_karabo_da}:daqOutput\""
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -454,16 +499,26 @@
"mod_dcs = {}\n",
"first_seq_raw = seq_files[0]\n",
"first_seq_corr = out_folder / first_seq_raw.name.replace(\"RAW\", \"CORR\")\n",
- "for mod in karabo_da:\n",
- " mod_dcs[mod] = {}\n",
- " with H5File(first_seq_corr) as out_dc:\n",
- " tid, mod_dcs[mod][\"train_corr_data\"] = next(\n",
- " out_dc[instr_mod_src, \"data.adc\"].trains()\n",
- " )\n",
+ "mod_dcs[corr_data_source] = {}\n",
+ "with H5File(first_seq_corr) as out_dc:\n",
+ " print(out_dc.all_sources)\n",
+ " tid, mod_dcs[corr_data_source][\"train_corr_data\"] = next(\n",
+ " out_dc[corr_data_source, \"data.adc\"].trains()\n",
+ " )\n",
+ "\n",
+ "if gh2_detector == \"25um\":\n",
+ " mod_dcs[corr_data_source][\"train_raw_data\"] = np.zeros((data_corr.shape[1], 1280 * 2), dtype=np.float32)\n",
+ " mod_dcs[corr_data_source][\"train_raw_gain\"] = np.zeros((data_corr.shape[1], 1280 * 2), dtype=np.uint8)\n",
+ "\n",
+ "for i, rec_mod in enumerate(receivers):\n",
" with H5File(first_seq_raw) as in_dc:\n",
- " train_dict = in_dc.train_from_id(tid)[1][instr_mod_src]\n",
- " mod_dcs[mod][\"train_raw_data\"] = train_dict[\"data.adc\"]\n",
- " mod_dcs[mod][\"train_raw_gain\"] = train_dict[\"data.gain\"]"
+ " train_dict = in_dc.train_from_id(tid)[1][rec_mod]\n",
+ " if gh2_detector == \"25um\":\n",
+ " mod_dcs[corr_data_source][\"train_raw_data\"][..., i::2] = train_dict[\"data.adc\"]\n",
+ " mod_dcs[corr_data_source][\"train_raw_gain\"][..., i::2] = train_dict[\"data.gain\"]\n",
+ " else:\n",
+ " mod_dcs[corr_data_source][\"train_raw_data\"] = train_dict[\"data.adc\"]\n",
+ " mod_dcs[corr_data_source][\"train_raw_gain\"] = train_dict[\"data.gain\"]"
]
},
{
@@ -477,27 +532,26 @@
"display(Markdown(f\"Train: {tid}\"))\n",
"\n",
"step_timer.start()\n",
- "for mod, pdu in zip(karabo_da, db_modules):\n",
"\n",
- " fig, ax = plt.subplots(figsize=(20, 10))\n",
- " raw_data = mod_dcs[mod][\"train_raw_data\"]\n",
- " im = ax.plot(np.mean(raw_data, axis=0))\n",
- " ax.set_title(f\"RAW module {mod} ({pdu})\")\n",
- " ax.set_xlabel(\"Strip #\", size=20)\n",
- " ax.set_ylabel(\"12-bit ADC output\", size=20)\n",
- " plt.xticks(fontsize=20)\n",
- " plt.yticks(fontsize=20)\n",
- " pass\n",
- "\n",
- " fig, ax = plt.subplots(figsize=(20, 10))\n",
- " corr_data = mod_dcs[mod][\"train_corr_data\"]\n",
- " im = ax.plot(np.mean(corr_data, axis=0))\n",
- " ax.set_title(f\"CORRECTED module {mod} ({pdu})\")\n",
- " ax.set_xlabel(\"Strip #\", size=20)\n",
- " ax.set_ylabel(\"10-bit KeV. output\", size=20)\n",
- " plt.xticks(fontsize=20)\n",
- " plt.yticks(fontsize=20)\n",
- " pass\n",
+ "fig, ax = plt.subplots(figsize=(20, 10))\n",
+ "raw_data = mod_dcs[corr_data_source][\"train_raw_data\"]\n",
+ "im = ax.plot(np.mean(raw_data, axis=0))\n",
+ "ax.set_title(f\"RAW module {receivers} ({db_modules})\")\n",
+ "ax.set_xlabel(\"Strip #\", size=20)\n",
+ "ax.set_ylabel(\"12-bit ADC output\", size=20)\n",
+ "plt.xticks(fontsize=20)\n",
+ "plt.yticks(fontsize=20)\n",
+ "pass\n",
+ "\n",
+ "fig, ax = plt.subplots(figsize=(20, 10))\n",
+ "corr_data = mod_dcs[corr_data_source][\"train_corr_data\"]\n",
+ "im = ax.plot(np.mean(corr_data, axis=0))\n",
+ "ax.set_title(f\"CORRECTED module {receivers} ({db_modules})\")\n",
+ "ax.set_xlabel(\"Strip #\", size=20)\n",
+ "ax.set_ylabel(\"10-bit KeV. output\", size=20)\n",
+ "plt.xticks(fontsize=20)\n",
+ "plt.yticks(fontsize=20)\n",
+ "pass\n",
"step_timer.done_step(\"Plotting mean data\")"
]
},
@@ -511,21 +565,20 @@
"display(Markdown(f\"### RAW and CORRECTED strips across pulses for train {tid}\"))\n",
"\n",
"step_timer.start()\n",
- "for mod, pdu in zip(karabo_da, db_modules):\n",
- " for plt_data, dname in zip(\n",
- " [\"train_raw_data\", \"train_corr_data\"], [\"RAW\", \"CORRECTED\"]\n",
- " ):\n",
- " fig, ax = plt.subplots(figsize=(15, 20))\n",
- " plt.rcParams.update({\"font.size\": 20})\n",
- "\n",
- " heatmapPlot(\n",
- " mod_dcs[mod][plt_data],\n",
- " y_label=\"Pulses\",\n",
- " x_label=\"Strips\",\n",
- " title=f\"{dname} module {mod} ({pdu})\",\n",
- " use_axis=ax,\n",
- " )\n",
- " pass\n",
+ "for plt_data, dname in zip(\n",
+ " [\"train_raw_data\", \"train_corr_data\"], [\"RAW\", \"CORRECTED\"]\n",
+ "):\n",
+ " fig, ax = plt.subplots(figsize=(15, 20))\n",
+ " plt.rcParams.update({\"font.size\": 20})\n",
+ "\n",
+ " heatmapPlot(\n",
+ " mod_dcs[corr_data_source][plt_data],\n",
+ " y_label=\"Pulses\",\n",
+ " x_label=\"Strips\",\n",
+ " title=f\"{dname} module {receivers} ({db_modules})\",\n",
+ " use_axis=ax,\n",
+ " )\n",
+ " pass\n",
"step_timer.done_step(\"Plotting RAW and CORRECTED data for one train\")"
]
},
@@ -603,9 +656,9 @@
],
"metadata": {
"kernelspec": {
- "display_name": "cal4_venv",
+ "display_name": ".cal_venv",
"language": "python",
- "name": "cal4_venv"
+ "name": "python3"
},
"language_info": {
"codemirror_mode": {
@@ -618,11 +671,6 @@
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.11"
- },
- "vscode": {
- "interpreter": {
- "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
- }
}
},
"nbformat": 4,