From 0c5eb525817d70aa84219448d9ee8bfe6df2b8c3 Mon Sep 17 00:00:00 2001
From: karnem <mikhail.karnevskiy@desy.de>
Date: Tue, 10 Mar 2020 15:25:41 +0100
Subject: [PATCH] Refactor FastCCD notebooks to use karabo_da
---
...s_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb | 38 +++++++-----
...orrectionNotebook_NewDAQ_FastCCD_NBC.ipynb | 58 +++++++++----------
2 files changed, 48 insertions(+), 48 deletions(-)
diff --git a/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb b/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
index 9ad805c32..6497c8ac4 100644
--- a/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
+++ b/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
@@ -19,25 +19,30 @@
"metadata": {},
"outputs": [],
"source": [
- "# Initial Parameters:\n",
- "\n",
+ "cluster_profile = \"noDB\" # ipcluster profile to use\n",
"in_folder = \"/gpfs/exfel/exp/SCS/201930/p900074/raw\" # input folder, required\n",
"out_folder = '/gpfs/exfel/exp/SCS/201930/p900074/scratch/' # output folder, required\n",
- "path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
- "path_inset = 'DA05'\n",
+ "sequence = 0 # sequence file to use\n",
"run = 351 # which run to read data from, required\n",
+ "\n",
+ "karabo_da = 'DA05' # data aggregators\n",
+ "karabo_id = \"SCS_CDIDET_FCCD2M\" # karabo prefix of PNCCD devices\n",
+ "receiver_id = \"FCCD\" # inset for receiver devices\n",
+ "path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
+ "h5path = '/INSTRUMENT/{}/DAQ/{}:daqOutput/data/image/pixels' # path to the data in the HDF5 file \n",
+ "h5path_t = '/CONTROL/{}/CTRL/LSLAN/inputA/crdg/value' # path to find temperature\n",
+ "h5path_cntrl = '/RUN/{}/DET/FCCD' # path to find control data\n",
+ "\n",
+ "use_dir_creation_date = True # To be used to retrieve calibration constants later on (for database time derivation)\n",
+ "cal_db_interface = \"tcp://max-exfl016:8020\" # the calibration database interface to use\n",
+ "cal_db_timeout = 300000 # timeout on calibration database requests\n",
+ "db_output = False # Output constants to the calibration database\n",
+ "local_output = True # output constants locally\n",
+ "\n",
"number_dark_frames = 0 # number of images to be used, if set to 0 all available images are used\n",
- "cluster_profile = \"noDB\" # ipcluster profile to use\n",
"# The two operation modes for FastCCD have fixed names which cannot be changed:\n",
"operation_mode = \"FF\" # FS stands for frame-store and FF for full-frame opeartion. \n",
- "h5path = '/INSTRUMENT/SCS_CDIDET_FCCD2M/DAQ/FCCD:daqOutput/data/image/pixels' # path to the data in the HDF5 file \n",
- "h5path_t = '/CONTROL/SCS_CDIDET_FCCD2M/CTRL/LSLAN/inputA/crdg/value' # path to find temperature\n",
- "h5path_cntrl = '/RUN/SCS_CDIDET_FCCD2M/DET/FCCD' # path to find control data\n",
- "cal_db_interface = \"tcp://max-exfl016:8020\" # the calibration database interface to use\n",
- "cal_db_timeout = 300000 # timeout on calibration database requests\n",
"temp_limits = 5 # to find calibration constants later on, the sensor temperature is allowed to vary by 5 units\n",
- "sequence = 0 # sequallence file to use\n",
- "use_dir_creation_date = True # To be used to retrieve calibration constants later on (for database time derivation)\n",
"bad_pixel_offset_sigma = 5. # Any pixel whose offset is beyond 5 standard deviations, is a bad pixel\n",
"bad_pixel_noise_sigma = 5. # Any pixel whose noise is beyond 5 standard deviations, is a bad pixel\n",
"sigmaNoise = 5. # Any pixel whose signal exceeds 'sigmaNoise'*noiseCM (common mode corrected noise) will be masked\n",
@@ -46,10 +51,8 @@
"chunkSize = 100 # Number of images to read per chunk\n",
"cpuCores = 40 # Specifies the number of running cpu cores\n",
"commonModeAxis = 1 # Axis along which common mode will be calculated (0: along rows, 1: along columns)\n",
- "\n",
"run_parallel = True # For parallel computation \n",
- "db_output = False # Output constants to the calibration database\n",
- "local_output = True # output constants locally\n",
+ "\n",
"\n",
"# According to our gain calibration using 55Fe source, we have the following conversion gains (e.g., 1 ADU = 6.3 e-):\n",
"ADU_to_electron_upper_hg = 6.3 # for upper hemisphere and high gain\n",
@@ -171,9 +174,12 @@
" print('\\nYou are analyzing data in FF mode.')\n",
" \n",
"ped_dir = \"{}/r{:04d}\".format(in_folder, run)\n",
- "fp_name = path_template.format(run, path_inset)\n",
+ "fp_name = path_template.format(run, karabo_da)\n",
"fp_path = '{}/{}'.format(ped_dir, fp_name)\n",
"filename = fp_path.format(sequence)\n",
+ "h5path = h5path.format(karabo_id, receiver_id)\n",
+ "h5path_t = h5path_t.format(karabo_id)\n",
+ "h5path_cntrl = h5path_cntrl.format(karabo_id)\n",
"\n",
"creation_time = None\n",
"if use_dir_creation_date:\n",
diff --git a/notebooks/FastCCD/CorrectionNotebook_NewDAQ_FastCCD_NBC.ipynb b/notebooks/FastCCD/CorrectionNotebook_NewDAQ_FastCCD_NBC.ipynb
index e6895334c..217325c9d 100644
--- a/notebooks/FastCCD/CorrectionNotebook_NewDAQ_FastCCD_NBC.ipynb
+++ b/notebooks/FastCCD/CorrectionNotebook_NewDAQ_FastCCD_NBC.ipynb
@@ -22,28 +22,34 @@
},
"outputs": [],
"source": [
+ "cluster_profile = \"noDB\" #ipcluster profile to use\n",
"in_folder = \"/gpfs/exfel/exp/SCS/201930/p900074/raw\" # input folder, required\n",
"out_folder = '/gpfs/exfel/data/scratch/karnem/test/fastccd' # output folder, required\n",
+ "sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
+ "run = 277 # run number\n",
+ "\n",
+ "karabo_da = 'DA05' # data aggregators\n",
+ "karabo_id = \"SCS_CDIDET_FCCD2M\" # karabo prefix of PNCCD devices\n",
+ "receiver_id = \"FCCD\" # inset for receiver devices\n",
"path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # path template in hdf5 file\n",
- "path_inset = 'DA05'\n",
- "run = 453 # run number\n",
- "h5path = '/INSTRUMENT/SCS_CDIDET_FCCD2M/DAQ/FCCD:daqOutput/data/image' # path in HDF5 file\n",
- "h5path_t = '/CONTROL/SCS_CDIDET_FCCD2M/CTRL/LSLAN/inputA/crdg/value' # temperature path in HDF5 file\n",
- "h5path_cntrl = '/RUN/SCS_CDIDET_FCCD2M/DET/FCCD' # path to control data\n",
- "cluster_profile = \"noDB\" #ipcluster profile to use\n",
+ "h5path = '/INSTRUMENT/{}/DAQ/{}:daqOutput/data/image' # path in HDF5 file\n",
+ "h5path_t = '/CONTROL/{}/CTRL/LSLAN/inputA/crdg/value' # temperature path in HDF5 file\n",
+ "h5path_cntrl = '/RUN/{}/DET/FCCD' # path to control data\n",
+ "\n",
+ "use_dir_creation_date = True # use dir creation data for calDB queries\n",
+ "cal_db_interface = \"tcp://max-exfl016:8015#8025\" # calibration DB interface to use\n",
+ "cal_db_timeout = 300000000 # timeout on caldb requests\n",
+ "\n",
+ "\n",
"cpuCores = 16 #Specifies the number of running cpu cores\n",
"operation_mode = \"FF\" # FS stands for frame-store and FF for full-frame opeartion\n",
"split_evt_primary_threshold = 7. # primary threshold for split event classification in terms of n sigma noise\n",
"split_evt_secondary_threshold = 4. # secondary threshold for split event classification in terms of n sigma noise\n",
"split_evt_mip_threshold = 1000. # MIP threshold for event classification\n",
- "cal_db_interface = \"tcp://max-exfl016:8015#8025\" # calibration DB interface to use\n",
- "cal_db_timeout = 300000000 # timeout on caldb requests\n",
- "sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"chunk_size_idim = 1 # H5 chunking size of output data\n",
"overwrite = True # overwrite existing files\n",
"sequences_per_node = 1 # sequences to correct per node\n",
"limit_images = 0 # limit images per file \n",
- "use_dir_creation_date = True # use dir creation data for calDB queries\n",
"time_offset_days = 0 # offset in days for calibration parameters\n",
"photon_energy_gain_map = 5.9 # energy in keV\n",
"fix_temperature = 0. # fix temperature to this value, set to 0 to use slow control value\n",
@@ -59,25 +65,9 @@
"correct_offset_drift = False # correct for offset drifts\n",
"do_pattern_classification = True # classify split events\n",
"\n",
- "def balance_sequences(in_folder, run, sequences, sequences_per_node, path_inset):\n",
- " import glob\n",
- " import re\n",
- " import numpy as np\n",
- " if sequences[0] == -1:\n",
- " sequence_files = glob.glob(\"{}/r{:04d}/*{}-S*.h5\".format(in_folder, run, path_inset))\n",
- " seq_nums = set()\n",
- " for sf in sequence_files:\n",
- " seqnum = re.findall(r\".*-S([0-9]*).h5\", sf)[0]\n",
- " seq_nums.add(int(seqnum))\n",
- " seq_nums -= set(sequences)\n",
- " nsplits = len(seq_nums)//sequences_per_node+1\n",
- " while nsplits > 8:\n",
- " sequences_per_node += 1\n",
- " nsplits = len(seq_nums)//sequences_per_node+1\n",
- " print(\"Changed to {} sequences per node to have a maximum of 8 concurrent jobs\".format(sequences_per_node))\n",
- " return [l.tolist() for l in np.array_split(list(seq_nums), nsplits)]\n",
- " else:\n",
- " return sequences"
+ "def balance_sequences(in_folder, run, sequences, sequences_per_node, karabo_da):\n",
+ " from xfel_calibrate.calibrate import balance_sequences as bs\n",
+ " return bs(in_folder, run, sequences, sequences_per_node, karabo_da)\n"
]
},
{
@@ -189,9 +179,13 @@
" print('\\nYou are analyzing data in FF mode.')\n",
" \n",
"ped_dir = \"{}/r{:04d}\".format(in_folder, run)\n",
- "fp_name = path_template.format(run, path_inset)\n",
+ "fp_name = path_template.format(run, karabo_da)\n",
"fp_path = '{}/{}'.format(ped_dir, fp_name)\n",
"\n",
+ "h5path = h5path.format(karabo_id, receiver_id)\n",
+ "h5path_t = h5path_t.format(karabo_id)\n",
+ "h5path_cntrl = h5path_cntrl.format(karabo_id)\n",
+ "\n",
"print(\"Reading data from: {}\\n\".format(fp_path))\n",
"print(\"Run is: {}\".format(run))\n",
"print(\"HDF5 path: {}\".format(h5path))\n",
@@ -277,14 +271,14 @@
" if sequences is None:\n",
" for seq in range(len(dirlist)):\n",
" \n",
- " if path_template.format(run, path_inset).format(seq) in abs_entry:\n",
+ " if path_template.format(run, karabo_da).format(seq) in abs_entry:\n",
" file_list.append(abs_entry)\n",
" total_sequences += 1\n",
" fsequences.append(seq)\n",
" else:\n",
" for seq in sequences:\n",
" \n",
- " if path_template.format(run, path_inset).format(seq) in abs_entry:\n",
+ " if path_template.format(run, karabo_da).format(seq) in abs_entry:\n",
" file_list.append(os.path.abspath(abs_entry))\n",
" total_sequences += 1\n",
" fsequences.append(seq)\n",
--
GitLab