diff --git a/notebooks/ePix/Characterize_Darks_ePix_NBC.ipynb b/notebooks/ePix/Characterize_Darks_ePix_NBC.ipynb
index 7b75803a015502c85de9dc424715818bb4eced71..420f0c03c2a8e5596a2c934eb9c9cc867b632081 100644
--- a/notebooks/ePix/Characterize_Darks_ePix_NBC.ipynb
+++ b/notebooks/ePix/Characterize_Darks_ePix_NBC.ipynb
@@ -19,27 +19,32 @@
"metadata": {},
"outputs": [],
"source": [
+ "cluster_profile = \"noDB\" # ipcluster profile to use\n",
"in_folder = '/gpfs/exfel/exp/MID/201930/p900071/raw' # input folder, required\n",
"out_folder = '/gpfs/exfel/exp/MID/201930/p900070/usr/dark/runs_4' # output folder, required\n",
- "path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
+ "sequence = 0 # sequence file to use\n",
"run = 466 # which run to read data from, required\n",
- "number_dark_frames = 0 # number of images to be used, if set to 0 all available images are used\n",
- "cluster_profile = \"noDB\" # ipcluster profile to use\n",
- "h5path = '/INSTRUMENT/{}/DET/RECEIVER:daqOutput/data/image/pixels' # path in the HDF5 file to images\n",
- "h5path_t = '/INSTRUMENT/{}/DET/RECEIVER:daqOutput/data/backTemp' # path to find temperature at\n",
+ "\n",
+ "karabo_id = \"MID_EXP_EPIX-1\" # karabo karabo_id\n",
+ "karabo_da = \"DA01\" # data aggregators\n",
+ "receiver_id = \"RECEIVER\" # inset for receiver devices\n",
+ "path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
+ "h5path = '/INSTRUMENT/{}/DET/{}:daqOutput/data/image/pixels' # path in the HDF5 file to images\n",
+ "h5path_t = '/INSTRUMENT/{}/DET/{}:daqOutput/data/backTemp' # path to find temperature at\n",
"h5path_cntrl = '/CONTROL/{}/DET' # path to control data\n",
+ "\n",
+ "use_dir_creation_date = True\n",
"cal_db_interface = \"tcp://max-exfl016:8020\" # calibration DB interface to use\n",
+ "cal_db_timeout = 300000 # timeout on caldb requests\n",
+ "db_output = False # Output constants to the calibration database\n",
+ "local_output = True # output constants locally\n",
+ "\n",
+ "number_dark_frames = 0 # number of images to be used, if set to 0 all available images are used\n",
"temp_limits = 5 # limit for parameter Operational temperature\n",
- "sequence = 0 # sequence file to use\n",
- "use_dir_creation_date = True\n",
- "db_module = 'ePix100_M15' # detector instance\n",
+ "db_module = 'ePix100_M15' # detector karabo_id\n",
"bias_voltage = 200 # bias voltage\n",
"in_vacuum = False # detector operated in vacuum\n",
- "instance = \"MID_EXP_EPIX-1\" # karabo instance\n",
- "path_inset = \"DA01\" # file inset for image data\n",
- "fix_temperature = 290. # fix temperature to this value\n",
- "db_output = False # Output constants to the calibration database\n",
- "local_output = True # output constants locally"
+ "fix_temperature = 290. # fix temperature to this value"
]
},
{
@@ -76,9 +81,9 @@
"import matplotlib.pyplot as plt\n",
"%matplotlib inline\n",
"\n",
- "h5path = h5path.format(instance)\n",
- "h5path_t = h5path_t.format(instance)\n",
- "h5path_cntrl = h5path_cntrl.format(instance)\n",
+ "h5path = h5path.format(karabo_id, receiver_id)\n",
+ "h5path_t = h5path_t.format(karabo_id, receiver_id)\n",
+ "h5path_cntrl = h5path_cntrl.format(karabo_id)\n",
"\n",
"def nImagesOrLimit(nImages, limit):\n",
" if limit == 0:\n",
@@ -100,7 +105,7 @@
"y = 768 # columns of the xPix100\n",
"\n",
"ped_dir = \"{}/r{:04d}\".format(in_folder, run)\n",
- "fp_name = path_template.format(run, path_inset).format(sequence)\n",
+ "fp_name = path_template.format(run, karabo_da).format(sequence)\n",
"filename = '{}/{}'.format(ped_dir, fp_name)\n",
"\n",
"print(\"Reading data from: {}\\n\".format(filename))\n",
@@ -290,7 +295,7 @@
" \n",
" if db_output:\n",
" try:\n",
- " metadata.send(cal_db_interface)\n",
+ " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
" print(\"Inject {} constants from {}\".format(const_name, \n",
" metadata.calibration_constant_version.begin_at))\n",
" except Exception as e:\n",
diff --git a/notebooks/ePix/Correction_ePix_NBC.ipynb b/notebooks/ePix/Correction_ePix_NBC.ipynb
index 9a5c64e12908fdca59172357baeca7749c542a99..3be55a3b1c3e82798ef00b97dfb52695d7cde4c3 100644
--- a/notebooks/ePix/Correction_ePix_NBC.ipynb
+++ b/notebooks/ePix/Correction_ePix_NBC.ipynb
@@ -22,30 +22,30 @@
},
"outputs": [],
"source": [
+ "cluster_profile = \"noDB\" # ipcluster profile to use\n",
"in_folder = \"/gpfs/exfel/exp/MID/201930/p900071/raw\" # input folder, required\n",
"out_folder = '/gpfs/exfel/data/scratch/karnem/test/' # output folder, required\n",
- "path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
+ "sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"run = 466 # which run to read data from, required\n",
- "h5path = '/INSTRUMENT/{}/DET/RECEIVER:daqOutput/data/image' # path in the HDF5 file to images\n",
- "h5path_t = '/INSTRUMENT/{}/DET/RECEIVER:daqOutput/data/backTemp' # path to find temperature at\n",
- "h5path_cntrl = '/CONTROL/{}/DET' # path to control data\n",
"\n",
- "path_inset = \"DA01\" # file inset for image data\n",
- "instance = \"MID_EXP_EPIX-1\" # karabo instance\n",
- "cluster_profile = \"noDB\" # ipcluster profile to use\n",
- "cpuCores = 4 # Specifies the number of running cpu cores\n",
+ "karabo_id = \"MID_EXP_EPIX-1\" # karabo karabo_id\n",
+ "karabo_da = \"DA01\" # data aggregators\n",
+ "receiver_id = \"RECEIVER\" # inset for receiver devices\n",
+ "path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
+ "h5path = '/INSTRUMENT/{}/DET/{}:daqOutput/data/image' # path in the HDF5 file to images\n",
+ "h5path_t = '/INSTRUMENT/{}/DET/{}:daqOutput/data/backTemp' # path to find temperature at\n",
+ "h5path_cntrl = '/CONTROL/{}/DET' # path to control data\n",
"\n",
+ "use_dir_creation_date = True # date constants injected before directory creation time\n",
"cal_db_interface = \"tcp://max-exfl016:8015#8025\" # calibration DB interface to use\n",
- "cal_db_timeout = 30000000 # timeout on caldb requests\n",
+ "cal_db_timeout = 300000 # timeout on caldb requests\n",
"\n",
- "sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
+ "cpuCores = 4 # Specifies the number of running cpu cores\n",
"chunk_size_idim = 1 # H5 chunking size of output data\n",
"overwrite = True # overwrite output folder\n",
"limit_images = 0 # process only first N images, 0 - process all\n",
- "use_dir_creation_date = True # date constants injected before directory creation time\n",
"db_module = \"ePix100_M15\" # module id in the database\n",
"sequences_per_node = 1 # number of sequence files per cluster node if run as slurm job, set to 0 to not run SLURM parallel\n",
- "\n",
"bias_voltage = 200 # bias voltage\n",
"in_vacuum = False # detector operated in vacuum\n",
"fix_temperature = 290. # fix temperature to this value\n",
@@ -56,25 +56,9 @@
"split_evt_secondary_threshold = 5. # secondary threshold for split event correction\n",
"split_evt_mip_threshold = 1000. # minimum ionizing particle threshold\n",
" \n",
- "def balance_sequences(in_folder, run, sequences, sequences_per_node):\n",
- " import glob\n",
- " import re\n",
- " import numpy as np\n",
- " if sequences[0] == -1:\n",
- " sequence_files = glob.glob(\"{}/r{:04d}/*-S*.h5\".format(in_folder, run))\n",
- " seq_nums = set()\n",
- " for sf in sequence_files:\n",
- " seqnum = re.findall(r\".*-S([0-9]*).h5\", sf)[0]\n",
- " seq_nums.add(int(seqnum))\n",
- " seq_nums -= set(sequences)\n",
- " else:\n",
- " seq_nums = set(sequences)\n",
- " nsplits = len(seq_nums)//sequences_per_node+1\n",
- " while nsplits > 8:\n",
- " sequences_per_node += 1\n",
- " nsplits = len(seq_nums)//sequences_per_node+1\n",
- " print(\"Changed to {} sequences per node to have a maximum of 8 concurrent jobs\".format(sequences_per_node))\n",
- " return [l.tolist() for l in np.array_split(list(seq_nums), nsplits) if l.size > 0]"
+ "def balance_sequences(in_folder, run, sequences, sequences_per_node, karabo_da):\n",
+ " from xfel_calibrate.calibrate import balance_sequences as bs\n",
+ " return bs(in_folder, run, sequences, sequences_per_node, karabo_da)\n"
]
},
{
@@ -120,9 +104,9 @@
"if sequences[0] == -1:\n",
" sequences = None\n",
"\n",
- "h5path = h5path.format(instance)\n",
- "h5path_t = h5path_t.format(instance)\n",
- "h5path_cntrl = h5path_cntrl.format(instance)\n",
+ "h5path = h5path.format(karabo_id, receiver_id)\n",
+ "h5path_t = h5path_t.format(karabo_id, receiver_id)\n",
+ "h5path_cntrl = h5path_cntrl.format(karabo_id)\n",
"\n",
"plot_unit = 'ADU'\n",
"if not no_relative_gain:\n",
@@ -146,7 +130,7 @@
"y = 768 # columns of the ePix100\n",
" \n",
"ped_dir = \"{}/r{:04d}\".format(in_folder, run)\n",
- "fp_name = path_template.format(run, path_inset)\n",
+ "fp_name = path_template.format(run, karabo_da)\n",
"fp_path = '{}/{}'.format(ped_dir, fp_name)\n",
"\n",
"print(\"Reading from: {}\".format(fp_path))\n",
@@ -225,14 +209,14 @@
" if sequences is None:\n",
" for seq in range(len(dirlist)):\n",
" \n",
- " if path_template.format(run, path_inset).format(seq) in abs_entry:\n",
+ " if path_template.format(run, karabo_da).format(seq) in abs_entry:\n",
" file_list.append(abs_entry)\n",
" total_sequences += 1\n",
" fsequences.append(seq)\n",
" else:\n",
" for seq in sequences:\n",
" \n",
- " if path_template.format(run, path_inset).format(seq) in abs_entry:\n",
+ " if path_template.format(run, karabo_da).format(seq) in abs_entry:\n",
" file_list.append(os.path.abspath(abs_entry))\n",
" total_sequences += 1\n",
" fsequences.append(seq)\n",