diff --git a/notebooks/LPD/Inject_calibration_constants_from_h5files.ipynb b/notebooks/LPD/Inject_calibration_constants_from_h5files.ipynb
index 1a2adc72dfe16697eb032d0f409b6330d1422316..2da696d06a67d80e82c21447917e4c09bd82e16e 100644
--- a/notebooks/LPD/Inject_calibration_constants_from_h5files.ipynb
+++ b/notebooks/LPD/Inject_calibration_constants_from_h5files.ipynb
@@ -13,13 +13,13 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# calibration constant parameters:\n",
- "constant_names = [\"GainMap\", \"GainAmpMap\", \"FFMap\"] # calibration constant name, required.\n",
- "in_folder = \"/gpfs/exfel/data/scratch/yousefh/LPD_CalinConst_Provisional/\" # calibration constant file, required.\n",
+ "constant_names = [\"\"] # calibration constant name, required.\n",
+ "in_folder = \"\" # calibration constants folder, required.\n",
"out_folder = \"\" # output folder to store report path in case the notebook is executed by CLI, required.\n",
"proposal = \"\" # Add proposal number to be sent to the database as a part of Raw data location.\n",
"runs = [\"\"] # Add list of runs to be sent to the database as a part of Raw data location.\n",
@@ -27,7 +27,6 @@
"# detector parameters:\n",
"karabo_id = \"FXE_DET_LPD1M-1\" # detector identifier.\n",
"karabo_da = [\"all\"] # karabo data aggregators. default \"all\" for all 16 karabo data aggregator names.\n",
- "modules = [-1] # List of module indicies to inject the constants for. default -1 for all 16 modules.\n",
"\n",
"# calibration database parameters:\n",
"cal_db_interface = \"tcp://max-exfl017:8015#8045\" # calibration DB zmq address.\n",
@@ -36,12 +35,14 @@
"memory_cells = 512 # Number of memory cells. Used for constant conditions.\n",
"bias_voltage = 250 # bias voltage value. Used for constant conditions.\n",
"capacitor = 5 # capacitor value. Used for constant conditions.\n",
+ "category = 0 # calibration constant source category, 0 for European XFEl and 1 for RAL. Used for constant conditions.\n",
+ "photon_energy = 9.2 # calibration constant photon energy. Used for constant conditions.\n",
"creation_time = '2020-01-20T14:12:06' # creation time for the injected constants. required format '2019-01-20T14:12:06'"
]
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -64,23 +65,21 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"pixels_x = pixels_y = 256\n",
"\n",
"if karabo_da[0].lower() == \"all\":\n",
- " if modules[0] == -1:\n",
- " modules = list(range(16))\n",
- " karabo_da = [f\"LPD{i:02d}\" for i in modules]\n",
+ " karabo_da = [f\"LPD{i:02d}\" for i in range(16)]\n",
"\n",
"# if proposal or runs are given assign file_loc \n",
"# for calibration constant versions metadata.\n",
"file_loc = \"\"\n",
"if proposal:\n",
" file_loc += f\"proposal:{proposal}\"\n",
- "if len(runs) > 0:\n",
+ "if runs[0] == \"\":\n",
" file_loc += f\"runs: {runs}\"\n",
"\n",
"if file_loc == \"\":\n",
@@ -92,7 +91,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -122,7 +121,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -165,90 +164,9 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "/gpfs/exfel/data/scratch/yousefh/LPD_CalinConst_Provisional 2020-01-20 14:12:06\n",
- "Physical detector units retrieved are: {'LPD00': 'LPD_SIV1_LPDV2_SM015', 'LPD01': 'LPD_SIV1_LPDV2_SM012', 'LPD02': 'LPD_SIV1_LPDV2_SM011', 'LPD03': 'LPD_SIV1_LPDV2_SM014', 'LPD04': 'LPD_SIV1_LPDV2_SM001', 'LPD05': None, 'LPD06': 'LPD_SIV1_LPDV2_SM004', 'LPD07': 'LPD_SIV1_LPDV2_SM002', 'LPD08': 'LPD_SIV1_LPDV2_SM009', 'LPD09': 'LPD_SIV1_LPDV2_SM006', 'LPD10': 'LPD_SIV1_LPDV2_SM007', 'LPD11': 'LPD_SIV1_LPDV2_SM005', 'LPD12': 'LPD_SIV1_LPDV2_SM000', 'LPD13': 'LPD_SIV1_LPDV2_SM010', 'LPD14': 'LPD_SIV1_LPDV2_SM008', 'LPD15': 'LPD_SIV1_LPDV2_SM013'} \n",
- "\n",
- "GainMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainAmpMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainAmpMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainAmpMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainAmpMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainAmpMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainAmpMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainAmpMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainAmpMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainAmpMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainAmpMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainAmpMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainAmpMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainAmpMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainAmpMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "GainAmpMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "FFMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n",
- "Can't decrement id ref count (write to backing store failed: time = Wed Apr 21 13:10:25 2021\n",
- ", filename = '/tmp/tmprj0cpuu3', file descriptor = 69, errno = 28, error message = 'No space left on device', ptr = 0x2b17524ae010, total write size = 498432672, bytes this sub-write = 498432672, bytes actually written = 18446744073709551615, offset = 306892800)\n",
- "\n",
- "GainMap is injected with creation-time: 2020-01-20 14:12:06\n",
- "\n"
- ]
- },
- {
- "ename": "KeyboardInterrupt",
- "evalue": "",
- "output_type": "error",
- "traceback": [
- "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
- "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
- "\u001b[0;32m<ipython-input-6-65188278bae1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 40\u001b[0m \u001b[0mprocess\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkarabo_da\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconstant_names\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 41\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mmultiprocessing\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPool\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocesses\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m16\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpool\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpool\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstarmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minject_constants\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
- "\u001b[0;32m/software/anaconda3/5.2/lib/python3.6/multiprocessing/pool.py\u001b[0m in \u001b[0;36mstarmap\u001b[0;34m(self, func, iterable, chunksize)\u001b[0m\n\u001b[1;32m 294\u001b[0m \u001b[0;31m`\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0mbecomes\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 295\u001b[0m '''\n\u001b[0;32m--> 296\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_map_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstarmapstar\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 297\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 298\u001b[0m def starmap_async(self, func, iterable, chunksize=None, callback=None,\n",
- "\u001b[0;32m/software/anaconda3/5.2/lib/python3.6/multiprocessing/pool.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 662\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 663\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 664\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 665\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mready\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 666\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
- "\u001b[0;32m/software/anaconda3/5.2/lib/python3.6/multiprocessing/pool.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 659\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 660\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 661\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_event\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 662\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 663\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
- "\u001b[0;32m/software/anaconda3/5.2/lib/python3.6/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 549\u001b[0m \u001b[0msignaled\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_flag\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 550\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msignaled\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 551\u001b[0;31m \u001b[0msignaled\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cond\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 552\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0msignaled\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 553\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
- "\u001b[0;32m/software/anaconda3/5.2/lib/python3.6/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 293\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 294\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 295\u001b[0;31m \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 296\u001b[0m \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 297\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
- "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
- ]
- }
- ],
+ "outputs": [],
"source": [
"c_folder, creation_time = validate_input_paths(in_folder, creation_time)\n",
"print(c_folder, creation_time)\n",
@@ -259,12 +177,14 @@
")\n",
"\n",
"# Calibration constants condition object.\n",
- "condition = Conditions.Dark.LPD(\n",
+ "condition = Conditions.Illuminated.LPD(\n",
" memory_cells=memory_cells,\n",
" bias_voltage=bias_voltage,\n",
+ " photon_energy=photon_energy,\n",
" pixels_x=pixels_x,\n",
" pixels_y=pixels_y,\n",
" capacitor=capacitor,\n",
+ " category=category,\n",
")\n",
"\n",
"# Retrieve all physical detector units for the given karabo_da list.\n",
@@ -276,37 +196,25 @@
" cal_db_interface=cal_db_interface,\n",
" snapshot_at=creation_time\n",
")\n",
+ "\n",
"mod_mapping = dict(zip(karabo_da, physical_units))\n",
"print(\"Physical detector units retrieved are: \", mod_mapping, \"\\n\")\n",
"\n",
"inp = []\n",
"for const in constant_names:\n",
" for k_da, pdu in mod_mapping.items():\n",
- " inp.append(\n",
- " (const,\n",
- " k_da,\n",
- " pdu,\n",
- " )\n",
- " )\n",
+ " inp.append((const, k_da, pdu))\n",
"\n",
- "process = int(len(karabo_da)*len(constant_names))\n",
"with multiprocessing.Pool(processes=16) as pool:\n",
" results = pool.starmap(inject_constants, inp)"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {
"kernelspec": {
- "display_name": "cal_venv",
+ "display_name": "Python 3",
"language": "python",
- "name": "cal_venv"
+ "name": "python3"
},
"language_info": {
"codemirror_mode": {
diff --git a/src/cal_tools/tools.py b/src/cal_tools/tools.py
index 7f6a1037839a98d6162feee0cfc12c1bd2ed6a8a..9021160723f536247e2e8dc42746ab7cf04b71a2 100644
--- a/src/cal_tools/tools.py
+++ b/src/cal_tools/tools.py
@@ -562,6 +562,8 @@ def get_from_db(karabo_id: str, karabo_da: str,
else:
return empty_constant, None
+already_printed = []
+
def send_to_db(db_module: str, karabo_id: str, constant, condition,
file_loc: str, report_path: str, cal_db_interface: str,
@@ -624,9 +626,11 @@ def send_to_db(db_module: str, karabo_id: str, constant, condition,
except Exception as e:
# TODO: refactor to use custom exception class
if "has already been taken" in str(e):
- print(f"WARNING: {constant.name} has already been "
- "injected with the same parameter "
- "conditions\n")
+ print(
+ f"WARNING: {constant.name} for {db_module}"
+ " has already been injected with the same "
+ "parameter conditions."
+ )
else:
print(f"{e}\n")
@@ -638,8 +642,11 @@ def send_to_db(db_module: str, karabo_id: str, constant, condition,
raise RuntimeError(f'{e}')
if success:
- print(f"{constant.name} is injected with creation-time: "
- f"{metadata.calibration_constant_version.begin_at}\n")
+ if constant.name not in already_printed:
+ print(
+ f"{constant.name} is injected with creation-time: "
+ f"{metadata.calibration_constant_version.begin_at}."
+ )
return metadata