From f01ae0c8b457c86666f54ae238b022bcbb1bdbf8 Mon Sep 17 00:00:00 2001
From: karnem <mikhail.karnevskiy@desy.de>
Date: Tue, 17 Mar 2020 10:29:30 +0100
Subject: [PATCH] Use karabo-da and karabo_id
---
...Jungfrau_Gain_Correct_and_Verify_NBC.ipynb | 65 +++++++-----
...rk_analysis_all_gains_burst_mode_NBC.ipynb | 100 +++++++++++-------
2 files changed, 99 insertions(+), 66 deletions(-)
diff --git a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
index 743df20a3..be535228d 100644
--- a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
@@ -17,38 +17,42 @@
"metadata": {},
"outputs": [],
"source": [
- "in_folder = \"/gpfs/exfel/exp/SPB/201922/p002566/raw\" # the folder to read data from, required\n",
- "run = 70 # runs to process, required\n",
+ "cluster_profile = \"noDB\" # cluster profile to use\n",
+ "in_folder = \"/gpfs/exfel/exp/CALLAB/202031/p900113/raw\" # the folder to read data from, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/karnem/test/005\" # the folder to output to, required\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
+ "run = 9999 # runs to process, required\n",
+ "\n",
+ "karabo_id = \"FXE_XAD_JF1M\" # karabo prefix of Jungfrau devices\n",
+ "karabo_da = 'JNGFR01' # data aggregators\n",
+ "receiver_id = \"RECEIVER-{}\" # inset for receiver devices\n",
+ "receiver_control_id = \"CONTROL\" # inset for control devices\n",
+ "path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # template to use for file name, double escape sequence number\n",
+ "h5path = '/INSTRUMENT/{}/DET/{}:daqOutput/data' # path in H5 file under which images are located\n",
+ "h5path_run = '/RUN/{}/DET/{}' # path to run data\n",
+ "h5path_cntrl = '/CONTROL/{}/DET/{}' # path to control data\n",
+ "karabo_id_control = \"\" # if control is on a different ID, set to empty string if it is the same a karabo-id\n",
+ "karabo_da_control = \"JNGFR01\" # file inset for control data\n",
+ "\n",
+ "use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
+ "cal_db_interface = \"tcp://max-exfl017:8016\" #\"tcp://max-exfl016:8015#8025\" # the database interface to use\n",
+ "cal_db_timeout = 180000 # timeout on caldb requests\",\n",
+ "\n",
"overwrite = True # set to True if existing data should be overwritten\n",
"no_relative_gain = False # do not do relative gain correction\n",
- "cluster_profile = \"noDB\" # cluster profile to use\n",
"bias_voltage = 180 # will be overwritten by value in file\n",
- "cal_db_interface = \"tcp://max-exfl016:8016\" #\"tcp://max-exfl016:8015#8025\" # the database interface to use\n",
- "cal_db_timeout = 180000 # timeout on caldb requests\",\n",
- "use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
"sequences_per_node = 5 # number of sequence files per cluster node if run as slurm job, set to 0 to not run SLURM parallel\n",
"photon_energy = 9.2 # photon energy in keV\n",
"chunk_size_idim = 1 # chunking size of imaging dimension, adjust if user software is sensitive to this.\n",
- "path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # template to use for file name, double escape sequence number\n",
- "path_template_control = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # template to use for file name, double escape sequence number\n",
"integration_time = 4.96 # integration time in us, will be overwritten by value in file\n",
- "h5path = '/INSTRUMENT/{}/DET/{}:daqOutput/data' # path in H5 file under which images are located\n",
"gmapfile = \"\" # variable is not used but left here for back compatibility\n",
- "karabo_id = \"SPB_IRDA_JNGFR\" # karabo prefix of Jungfrau devices\n",
- "karabo_id_control = \"\" # if control is on a different ID, set to empty string for using the same as for data\n",
- "receiver_id = \"MODULE_3\" # inset for receiver devices\n",
- "control_id = \"CONTROL\" # inset for control devices\n",
- "db_module = \"Jungfrau_M273\" # ID of module in calibration database\n",
- "path_inset = \"JNGFR03\" # file inset for image data\n",
- "path_inset_control = \"JNGFR01\" # file inset for control data\n",
+ "db_module = [\"Jungfrau_M233\"] # ID of module in calibration database\n",
"manual_slow_data = False # if true, use manually entered bias_voltage and integration_time values\n",
"chunk_size = 0\n",
"\n",
- "def balance_sequences(in_folder, run, sequences, sequences_per_node, path_inset):\n",
+ "def balance_sequences(in_folder, run, sequences, sequences_per_node, karabo_da):\n",
" from xfel_calibrate.calibrate import balance_sequences as bs\n",
- " return bs(in_folder, run, sequences, sequences_per_node, path_inset)"
+ " return bs(in_folder, run, sequences, sequences_per_node, karabo_da)"
]
},
{
@@ -99,9 +103,17 @@
"view = client[:]\n",
"view.use_dill()\n",
"\n",
+ "receiver_id = receiver_id.format(int(karabo_da[-2:]))\n",
"h5path = h5path.format(karabo_id, receiver_id)\n",
"ped_dir = \"{}/r{:04d}\".format(in_folder, run)\n",
"\n",
+ "# TODO\n",
+ "# this trick is needed until proper mapping is introduced\n",
+ "if len(db_module)>1:\n",
+ " db_module = db_module[int(karabo_da[-2:])-1]\n",
+ "else:\n",
+ " db_module = db_module[0]\n",
+ " \n",
"if ped_dir[-1] == \"/\":\n",
" ped_dir = ped_dir[:-1]\n",
"\n",
@@ -111,10 +123,10 @@
" raise AttributeError(\"Output path exists! Exiting\") \n",
"\n",
"\n",
- "fp_name = path_template.format(run, path_inset)\n",
+ "fp_name = path_template.format(run, karabo_da)\n",
"fp_path = '{}/{}'.format(ped_dir, fp_name)\n",
"\n",
- "fp_name_contr = path_template_control.format(run, path_inset_control)\n",
+ "fp_name_contr = path_template.format(run, karabo_da_control)\n",
"fp_path_contr = '{}/{}'.format(ped_dir, fp_name_contr)\n",
"\n",
"\n",
@@ -162,11 +174,12 @@
"source": [
"if not manual_slow_data:\n",
" with h5py.File(fp_path_contr.format(0), 'r') as f:\n",
- " integration_time = float(f['/RUN/{}/DET/{}/exposureTime/value'.format(karabo_id_control, control_id)][()]*1e6)\n",
- " bias_voltage = int(np.squeeze(f['/RUN/{}/DET/{}/vHighVoltage/value'.format(karabo_id_control, control_id)])[0])\n",
+ " run_path = h5path_run.format(karabo_id_control, receiver_control_id)\n",
+ " integration_time = float(f[f'{run_path}/exposureTime/value'][()]*1e6)\n",
+ " bias_voltage = int(np.squeeze(f[f'{run_path}/vHighVoltage/value'])[0])\n",
" \n",
" \n",
- "control_path = '/CONTROL/{}/DET/{}'.format(karabo_id_control, control_id)\n",
+ "control_path = h5path_cntrl.format(karabo_id_control, receiver_control_id)\n",
"this_run_mcells, sc_start = check_memoryCells(fp_path_contr.format(0), control_path)\n",
"if this_run_mcells == 1:\n",
" memoryCells = 1\n",
@@ -198,13 +211,13 @@
" if sequences is None:\n",
" for seq in range(len(dirlist)):\n",
" \n",
- " if path_template.format(run, path_inset).format(seq) in abs_entry:\n",
+ " if path_template.format(run, karabo_da).format(seq) in abs_entry:\n",
" file_list.append(abs_entry)\n",
" total_sequences += 1\n",
" fsequences.append(seq)\n",
" else:\n",
" for seq in sequences:\n",
- " if path_template.format(run, path_inset).format(seq) in abs_entry:\n",
+ " if path_template.format(run, karabo_da).format(seq) in abs_entry:\n",
" file_list.append(os.path.abspath(abs_entry))\n",
" total_sequences += 1\n",
" fsequences.append(seq)\n",
@@ -651,7 +664,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.7"
+ "version": "3.7.6"
}
},
"nbformat": 4,
diff --git a/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
index 3c6e1a67d..3401a39b8 100644
--- a/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
@@ -17,14 +17,31 @@
"metadata": {},
"outputs": [],
"source": [
- "in_folder = '/gpfs/exfel/exp/FXE/201931/p900088/raw/' # folder under which runs are located, required\n",
+ "cluster_profile = 'noDB' # the ipcluster profile name\n",
+ "in_folder = '/gpfs/exfel/exp/FXE/201931/p900089/raw/' # folder under which runs are located, required\n",
"out_folder = '/gpfs/exfel/data/scratch/karnem/test_dark/' # path to place reports at, required\n",
"sequences = 1 # number of sequence files in that run\n",
+ "run_high = 86 # run number for G0 dark run, required\n",
+ "run_med = 87 # run number for G1 dark run, required\n",
+ "run_low = 88 # run number for G2 dark run, required\n",
+ "\n",
+ "karabo_da = ['JNGFR01'] # list of data aggregators, which corresponds to different JF modules\n",
+ "karabo_id = \"FXE_XAD_JF1M\" # bla karabo prefix of Jungfrau devices\n",
+ "karabo_id_control = \"\" # if control is on a different ID, set to empty string if it is the same a karabo-id\n",
+ "receiver_id = 'RECEIVER-{}' # inset for receiver devices\n",
+ "receiver_control_id = \"CONTROL\" # inset for control devices\n",
"path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # template to use for file name, double escape sequence number\n",
- "path_inset = \"JNGFR02\" # file inset for image data\n",
- "path_inset_control = \"JNGFR02\" # file inset for control data\n",
- "cluster_profile = 'noDB' # the ipcluster profile name\n",
+ "h5path = '/INSTRUMENT/{}/DET/{}:daqOutput/data' # path in H5 file under which images are located\n",
+ "h5path_run = '/RUN/{}/DET/{}' # path to run data\n",
+ "h5path_cntrl = '/CONTROL/{}/DET/{}' # path to control data\n",
+ "karabo_da_control = \"JNGFR01\" # file inset for control data\n",
+ "\n",
+ "use_dir_creation_date = True # use dir creation date\n",
"cal_db_interface = 'tcp://max-exfl016:8016' # calibrate db interface to connect to\n",
+ "cal_db_timeout = 300000 # timeout on caldb requests\n",
+ "local_output = True # output constants locally\n",
+ "db_output = False # output constants to database\n",
+ "\n",
"integration_time = 1000 # integration time in us, will be overwritten by value in file\n",
"bias_voltage = 90 # sensor bias voltage in V, will be overwritten by value in file\n",
"badpixel_threshold_sigma = 20. # bad pixels defined by values outside n times this std from median\n",
@@ -33,26 +50,17 @@
"chunkSize = 10 # iteration chunk size, needs to match or be less than number of images in a sequence file\n",
"imageRange = [0, 500] # image range in which to evaluate\n",
"memoryCells = 16 # number of memory cells\n",
- "h5path = '/INSTRUMENT/{}/DET/{}:daqOutput/data' # path in H5 file under which images are located\n",
- "run_high = 144 # run number for G0 dark run, required\n",
- "run_med = 145 # run number for G1 dark run, required\n",
- "run_low = 146 # run number for G2 dark run, required\n",
- "karabo_id = \"FXE_XAD_JF500K\" # karabo prefix of Jungfrau devices\n",
- "karabo_id_control = \"\" # if control is on a different ID, set to empty string for using the same as for data\n",
- "receiver_id = \"RECEIVER\" # inset for receiver devices\n",
- "control_id = \"CONTROL\" # inset for control devices\n",
- "db_module = \"Jungfrau_M260\" # ID of module in calibration database\n",
- "use_dir_creation_date = True # use dir creation date\n",
+ "db_module = [\"Jungfrau_M260\"] # ID of module in calibration database\n",
"manual_slow_data = False # if true, use manually entered bias_voltage and integration_time values\n",
- "time_limits = 0.025 # to find calibration constants later on, the integration time is allowed to vary by 0.5 us\n",
- "local_output = True # output constants locally\n",
- "db_output = False # output constants to database"
+ "time_limits = 0.025 # to find calibration constants later on, the integration time is allowed to vary by 0.5 us\n"
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "code_folding": []
+ },
"outputs": [],
"source": [
"import warnings\n",
@@ -83,6 +91,28 @@
"from XFELDetAna.plotting.histogram import histPlot\n",
"from XFELDetAna.plotting.heatmap import heatmapPlot\n",
"\n",
+ "import os\n",
+ "os.makedirs(out_folder, exist_ok=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "path_inset = karabo_da[0] # karabo_da is a concurrency parameter\n",
+ "receiver_id = receiver_id.format(int(path_inset[-2:]))\n",
+ "proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
+ "file_loc = 'proposal:{} runs:{} {} {}'.format(proposal, run_high, run_med, run_low)\n",
+ "\n",
+ "# TODO \n",
+ "# this trick is needed until proper mapping is introduced\n",
+ "if len(db_module)>1:\n",
+ " db_module = db_module[int(path_inset[-2:])-1]\n",
+ "else:\n",
+ " db_module = db_module[0]\n",
+ " \n",
"# Constants relevant for the analysis\n",
"run_nums = [run_high, run_med, run_low] # run number for G0/HG0, G1, G2 \n",
"sensorSize = [1024, 512]\n",
@@ -104,19 +134,9 @@
"\n",
"if karabo_id_control == \"\":\n",
" karabo_id_control = karabo_id\n",
- " \n",
- "import os\n",
- "os.makedirs(out_folder, exist_ok=True)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
- "file_loc = 'proposal:{} runs:{} {} {}'.format(proposal, run_high, run_med, run_low)"
+ "\n",
+ "print('Path inset ', path_inset)\n",
+ "print('Receiver Id ', receiver_id)"
]
},
{
@@ -162,20 +182,21 @@
" n_empty_sc = 0\n",
" \n",
" ped_dir = \"{}/r{:04d}/\".format(in_folder, r_n)\n",
- " fp_name = path_template.format(r_n, path_inset_control)\n",
+ " fp_name = path_template.format(r_n, karabo_da_control)\n",
" fp_path = '{}/{}'.format(ped_dir, fp_name)\n",
" \n",
" n_files = len(glob.glob(\"{}/*{}*.h5\".format(ped_dir, path_inset)))\n",
" myRange = range(0, n_files)\n",
- " control_path = '/CONTROL/{}/DET/{}'.format(karabo_id_control, control_id)\n",
+ " control_path = h5path_cntrl.format(karabo_id_control, receiver_control_id)\n",
" \n",
" this_run_mcells, sc_start = check_memoryCells(fp_path.format(0).format(myRange[0]), control_path)\n",
" \n",
" if noise_map is None:\n",
" if not manual_slow_data:\n",
" with h5py.File(fp_path.format(0), 'r') as f:\n",
- " integration_time = float(f['/RUN/{}/DET/{}/exposureTime/value'.format(karabo_id_control, control_id)][()]*1e6)\n",
- " bias_voltage = int(np.squeeze(f['/RUN/{}/DET/{}/vHighVoltage/value'.format(karabo_id_control, control_id)])[0])\n",
+ " run_path = h5path_run.format(karabo_id_control, receiver_control_id)\n",
+ " integration_time = float(f[f'{run_path}/exposureTime/value'][()]*1e6)\n",
+ " bias_voltage = int(np.squeeze(f[f'{run_path}/vHighVoltage/value'])[0])\n",
" print(\"Integration time is {} us\".format(integration_time))\n",
" print(\"Bias voltage is {} V\".format(bias_voltage))\n",
" if this_run_mcells == 1:\n",
@@ -279,14 +300,13 @@
"#%matplotlib notebook\n",
"\n",
"from XFELDetAna.plotting.histogram import histPlot\n",
- "from XFELDetAna.plotting.heatmap import heatmapPlot \n",
- "\n",
+ "from XFELDetAna.plotting.heatmap import heatmapPlot\n",
"\n",
"g_name = ['G0', 'G1', 'G2']\n",
"g_range = [(0, 8000), (8000, 16000), (8000, 16000)]\n",
"n_range = [(0., 50.), (0., 50.), (0., 50.)]\n",
"\n",
- "unit = '[ADCu]'\n"
+ "unit = '[ADCu]'"
]
},
{
@@ -454,7 +474,7 @@
" \n",
" metadata.calibration_constant_version.raw_data_location = file_loc\n",
" if db_output:\n",
- " metadata.send(cal_db_interface)\n",
+ " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
" print('Constants {} is sent to the data base'.format(key))\n",
" \n",
" # save everything to file.\n",
@@ -496,7 +516,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.7"
+ "version": "3.7.6"
}
},
"nbformat": 4,
--
GitLab