diff --git a/notebooks/ePix/Characterize_Darks_ePix10K_NBC.ipynb b/notebooks/ePix/Characterize_Darks_ePix10K_NBC.ipynb
index 8d2b228d076c9219b679da8cffb1ea25ea12dab3..29a5d8d7a04bc1a1fa2c5eeb9a0657349c306fac 100644
--- a/notebooks/ePix/Characterize_Darks_ePix10K_NBC.ipynb
+++ b/notebooks/ePix/Characterize_Darks_ePix10K_NBC.ipynb
@@ -19,27 +19,32 @@
"metadata": {},
"outputs": [],
"source": [
+ "cluster_profile = \"noDB\" # ipcluster profile to use\n",
"in_folder = '/gpfs/exfel/exp/HED/201922/p002550/raw' # input folder, required\n",
"out_folder = '/gpfs/exfel/exp/HED/201922/p002550/usr/dark/' # output folder, required\n",
- "path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
+ "sequence = 0 # sequence file to use\n",
"run = 55 # which run to read data from, required\n",
- "number_dark_frames = 0 # number of images to be used, if set to 0 all available images are used\n",
- "cluster_profile = \"noDB\" # ipcluster profile to use\n",
- "h5path = '/INSTRUMENT/{}/DET/RECEIVER:daqOutput/data/image/pixels' # path in the HDF5 file to images\n",
- "h5path_t = '/INSTRUMENT/{}/DET/RECEIVER:daqOutput/data/backTemp' # path to find temperature at\n",
+ "\n",
+ "karabo_id = \"HED_IA1_EPIX10K-1\" # karabo karabo_id\n",
+ "karabo_da = \"EPIX03\" # data aggregators\n",
+ "receiver_id = \"RECEIVER\" # inset for receiver devices\n",
+ "path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
+ "h5path = '/INSTRUMENT/{}/DET/{}:daqOutput/data/image/pixels' # path in the HDF5 file to images\n",
+ "h5path_t = '/INSTRUMENT/{}/DET/{}:daqOutput/data/backTemp' # path to find temperature at\n",
"h5path_cntrl = '/CONTROL/{}/DET' # path to control data\n",
+ "\n",
+ "use_dir_creation_date = True\n",
"cal_db_interface = \"tcp://max-exfl016:8020\" # calibration DB interface to use\n",
+ "cal_db_timeout = 300000 # timeout on caldb requests\n",
+ "db_output = False # Output constants to the calibration database\n",
+ "local_output = True # output constants locally\n",
+ "\n",
"temp_limits = 5 # limit for parameter Operational temperature\n",
- "sequence = 0 # sequence file to use\n",
- "use_dir_creation_date = True\n",
- "db_module = 'ePix10K_M43' # detector instance\n",
+ "number_dark_frames = 0 # number of images to be used, if set to 0 all available images are used\n",
+ "db_module = 'ePix10K_M43' # detector karabo_id\n",
"bias_voltage = 200 # bias voltage\n",
"in_vacuum = False # detector operated in vacuum\n",
- "instance = \"HED_IA1_EPIX10K-1\" # karabo instance\n",
- "path_inset = \"EPIX03\" # file inset for image data\n",
- "fix_temperature = 290. # fix temperature to this value\n",
- "db_output = False # Output constants to the calibration database\n",
- "local_output = True # output constants locally"
+ "fix_temperature = 290. # fix temperature to this value"
]
},
{
@@ -77,9 +82,9 @@
"import matplotlib.pyplot as plt\n",
"%matplotlib inline\n",
"\n",
- "h5path = h5path.format(instance)\n",
- "h5path_t = h5path_t.format(instance)\n",
- "h5path_cntrl = h5path_cntrl.format(instance)\n",
+ "h5path = h5path.format(karabo_id, receiver_id)\n",
+ "h5path_t = h5path_t.format(karabo_id, receiver_id)\n",
+ "h5path_cntrl = h5path_cntrl.format(karabo_id)\n",
"\n",
"def nImagesOrLimit(nImages, limit):\n",
" if limit == 0:\n",
@@ -101,8 +106,7 @@
"y = 384 # columns of the ePix10K\n",
"\n",
"ped_dir = \"{}/r{:04d}\".format(in_folder, run)\n",
- "out_folder = \"{}/r{:04d}\".format(out_folder, run)\n",
- "fp_name = path_template.format(run, path_inset).format(sequence)\n",
+ "fp_name = path_template.format(run, karabo_da).format(sequence)\n",
"filename = '{}/{}'.format(ped_dir, fp_name)\n",
"\n",
"print(\"Reading data from: {}\\n\".format(filename))\n",
@@ -149,8 +153,7 @@
" temperature_k))\n",
" print(\"Operated in vacuum: {} \".format(in_vacuum))\n",
" \n",
- "if not os.path.exists(out_folder):\n",
- " os.makedirs(out_folder)"
+ "os.makedirs(out_path, exist_ok=True)"
]
},
{
@@ -299,7 +302,7 @@
" \n",
" if db_output:\n",
" try:\n",
- " metadata.send(cal_db_interface)\n",
+ " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
" print(\"Inject {} constants from {}\".format(const_name, \n",
" metadata.calibration_constant_version.begin_at))\n",
" except Exception as e:\n",
diff --git a/notebooks/ePix/Correction_ePix10K_NBC.ipynb b/notebooks/ePix/Correction_ePix10K_NBC.ipynb
index f4a3517b2a8d0b4ca994dd8c77bb11558afbb6a0..8a49dd47ffe08bdd94a5a49f3e6a24539d9bea71 100644
--- a/notebooks/ePix/Correction_ePix10K_NBC.ipynb
+++ b/notebooks/ePix/Correction_ePix10K_NBC.ipynb
@@ -22,30 +22,30 @@
},
"outputs": [],
"source": [
+ "cluster_profile = \"noDB\" # ipcluster profile to use\n",
"in_folder = \"/gpfs/exfel/exp/HED/201922/p002550/raw\" # input folder, required\n",
"out_folder = '/gpfs/exfel/data/scratch/karnem/test/' # output folder, required\n",
- "path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
+ "sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"run = 55 # which run to read data from, required\n",
- "h5path = '/INSTRUMENT/{}/DET/RECEIVER:daqOutput/data/image' # path in the HDF5 file to images\n",
- "h5path_t = '/INSTRUMENT/{}/DET/RECEIVER:daqOutput/data/backTemp' # path to find temperature at\n",
- "h5path_cntrl = '/CONTROL/{}/DET' # path to control data\n",
"\n",
- "path_inset = \"EPIX03\" # file inset for image data\n",
- "instance = \"HED_IA1_EPIX10K-1\" # karabo instance\n",
- "cluster_profile = \"noDB\" # ipcluster profile to use\n",
- "cpuCores = 4 # Specifies the number of running cpu cores\n",
+ "karabo_id = \"HED_IA1_EPIX10K-1\" # karabo karabo_id\n",
+ "karabo_da = \"EPIX03\" # data aggregators\n",
+ "receiver_id = \"RECEIVER\" # inset for receiver devices\n",
+ "path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
+ "h5path = '/INSTRUMENT/{}/DET/{}:daqOutput/data/image' # path in the HDF5 file to images\n",
+ "h5path_t = '/INSTRUMENT/{}/DET/{}:daqOutput/data/backTemp' # path to find temperature at\n",
+ "h5path_cntrl = '/CONTROL/{}/DET' # path to control data\n",
"\n",
+ "use_dir_creation_date = True # date constants injected before directory creation time\n",
"cal_db_interface = \"tcp://max-exfl016:8015#8025\" # calibration DB interface to use\n",
"cal_db_timeout = 30000000 # timeout on caldb requests\n",
"\n",
- "sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
+ "cpuCores = 4 # Specifies the number of running cpu cores\n",
"chunk_size_idim = 1 # H5 chunking size of output data\n",
"overwrite = True # overwrite output folder\n",
"limit_images = 0 # process only first N images, 0 - process all\n",
- "use_dir_creation_date = True # date constants injected before directory creation time\n",
"db_module = \"ePix10K_M43\" # module id in the database\n",
"sequences_per_node = 1 # number of sequence files per cluster node if run as slurm job, set to 0 to not run SLURM parallel\n",
- "\n",
"bias_voltage = 200 # bias voltage\n",
"in_vacuum = False # detector operated in vacuum\n",
"fix_temperature = 290. # fix temperature to this value\n",
@@ -53,25 +53,9 @@
"split_evt_secondary_threshold = 5. # secondary threshold for split event correction\n",
"split_evt_mip_threshold = 1000. # minimum ionizing particle threshold\n",
" \n",
- "def balance_sequences(in_folder, run, sequences, sequences_per_node):\n",
- " import glob\n",
- " import re\n",
- " import numpy as np\n",
- " if sequences[0] == -1:\n",
- " sequence_files = glob.glob(\"{}/r{:04d}/*-S*.h5\".format(in_folder, run))\n",
- " seq_nums = set()\n",
- " for sf in sequence_files:\n",
- " seqnum = re.findall(r\".*-S([0-9]*).h5\", sf)[0]\n",
- " seq_nums.add(int(seqnum))\n",
- " seq_nums -= set(sequences)\n",
- " else:\n",
- " seq_nums = set(sequences)\n",
- " nsplits = len(seq_nums)//sequences_per_node+1\n",
- " while nsplits > 8:\n",
- " sequences_per_node += 1\n",
- " nsplits = len(seq_nums)//sequences_per_node+1\n",
- " print(\"Changed to {} sequences per node to have a maximum of 8 concurrent jobs\".format(sequences_per_node))\n",
- " return [l.tolist() for l in np.array_split(list(seq_nums), nsplits) if l.size > 0]"
+ "def balance_sequences(in_folder, run, sequences, sequences_per_node, karabo_da):\n",
+ " from xfel_calibrate.calibrate import balance_sequences as bs\n",
+ " return bs(in_folder, run, sequences, sequences_per_node, karabo_da)"
]
},
{
@@ -117,9 +101,9 @@
"if sequences[0] == -1:\n",
" sequences = None\n",
"\n",
- "h5path = h5path.format(instance)\n",
- "h5path_t = h5path_t.format(instance)\n",
- "h5path_cntrl = h5path_cntrl.format(instance)"
+ "h5path = h5path.format(karabo_id, receiver_id)\n",
+ "h5path_t = h5path_t.format(karabo_id, receiver_id)\n",
+ "h5path_cntrl = h5path_cntrl.format(karabo_id)"
]
},
{
@@ -137,7 +121,7 @@
"y = 384 # columns of the ePix10K\n",
" \n",
"ped_dir = \"{}/r{:04d}\".format(in_folder, run)\n",
- "fp_name = path_template.format(run, path_inset)\n",
+ "fp_name = path_template.format(run, karabo_da)\n",
"fp_path = '{}/{}'.format(ped_dir, fp_name)\n",
"\n",
"print(\"Reading from: {}\".format(fp_path))\n",
@@ -218,14 +202,14 @@
" if sequences is None:\n",
" for seq in range(len(dirlist)):\n",
" \n",
- " if path_template.format(run, path_inset).format(seq) in abs_entry:\n",
+ " if path_template.format(run, karabo_da).format(seq) in abs_entry:\n",
" file_list.append(abs_entry)\n",
" total_sequences += 1\n",
" fsequences.append(seq)\n",
" else:\n",
" for seq in sequences:\n",
" \n",
- " if path_template.format(run, path_inset).format(seq) in abs_entry:\n",
+ " if path_template.format(run, karabo_da).format(seq) in abs_entry:\n",
" file_list.append(os.path.abspath(abs_entry))\n",
" total_sequences += 1\n",
" fsequences.append(seq)\n",