diff --git a/bin/activate.sh b/bin/activate.sh
index 9584c3b78f61ec4a2c565e277aee4dafae1ee9de..8c8a0500e9d400a6094aaf05192ea40fbc5bde96 100644
--- a/bin/activate.sh
+++ b/bin/activate.sh
@@ -1,5 +1,5 @@
source /etc/profile.d/modules.sh
module load anaconda/3
-module load texlive
+module load texlive/2019
# export path to python environment
-export PATH=/home/${USER}/.local/bin:$PATH
+export PATH=$HOME/.local/bin:$PATH
diff --git a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
index d0130889643859dc904448f39d0346ae71636ff7..ec232229b6f13f75d2f51be924c9c2d56c22e42d 100644
--- a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
+++ b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
@@ -56,7 +56,7 @@
"number_dark_frames = 0 # number of images to be used, if set to 0 all available images are used\n",
"chunkSize = 100 # number of images to read per chunk\n",
"fix_temperature = 0. # fix temperature in K, set to 0. to use value from slow data\n",
- "gain = 0. # the detector's gain setting, It is later read from file and this value is overwritten\n",
+ "gain = 1 # the detector's gain setting, It is later read from file and this value is overwritten\n",
"bias_voltage = 0. # the detector's bias voltage. set to 0. to use value from slow data.\n",
"integration_time = 70 # detector's integration time\n",
"commonModeAxis = 0 # axis along which common mode will be calculated (0: along rows, 1: along columns)\n",
@@ -206,7 +206,6 @@
" try:\n",
" with h5py.File(ctrl_fname, \"r\") as f:\n",
" if bias_voltage == 0.:\n",
- " print(\"bias voltage control h5path:\", os.path.join(mdl_ctrl_path, \"DAQ_MPOD/u0voltage/value\"))\n",
" bias_voltage = abs(f[os.path.join(mdl_ctrl_path, \"DAQ_MPOD/u0voltage/value\")][0])\n",
" gain = f[os.path.join(mdl_ctrl_path, \"DAQ_GAIN/pNCCDGain/value\")][0]\n",
" if fix_temperature == 0.:\n",
diff --git a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
index b1484e339ed14465b720bee8c84e7a500fd416a5..18c45d5bd17fc41c4b881ecbe37fcfd30e2facac 100644
--- a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
+++ b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
@@ -22,10 +22,10 @@
},
"outputs": [],
"source": [
- "in_folder = \"/gpfs/exfel/exp/SQS/202031/p900166/raw\" # input folder\n",
- "out_folder = '/gpfs/exfel/data/scratch/ahmedk/test/pnccd' # output folder\n",
- "run = 39 # which run to read data from\n",
- "sequences = [0] # sequences to correct, set to -1 for all, range allowed\n",
+ "in_folder = \"/gpfs/exfel/exp/SQS/202022/p002720/raw\" # input folder\n",
+ "out_folder = '/gpfs/exfel/data/scratch/setoodeh' # output folder\n",
+ "run = 53 # which run to read data from\n",
+ "sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"\n",
"db_module = \"pnCCD_M205_M206\"\n",
"karabo_da = 'PNCCD01' # data aggregators\n",
@@ -58,7 +58,7 @@
"gain = 0. # the detector's gain setting, It is later read from file and this value is overwritten\n",
"bias_voltage = 0. # the detector's bias voltage. set to 0. to use value from slow data.\n",
"integration_time = 70\n",
- "photon_energy = 1.6 # Al fluorescence in keV\n",
+ "photon_energy = 1.5 # Al fluorescence in keV\n",
"\n",
"cal_db_interface = \"tcp://max-exfl016:8015\" # calibration DB interface to use\n",
"cal_db_timeout = 300000 # timeout on caldb requests\n",
@@ -68,7 +68,7 @@
"common_mode = True # Apply common mode correction\n",
"relgain = True # Apply relative gain correction\n",
"cti = False # Apply charge transfer inefficiency correction (not implemented, yet)\n",
- "do_pattern_classification = True # classify split events"
+ "do_pattern_classification = False # classify split events"
]
},
{
@@ -311,14 +311,14 @@
"# For all xcal histograms:\n",
"if gain == 1:\n",
" Hist_Bin_Dict = {\n",
- " \"bins\": 70000, # number of bins \n",
- " \"bin_range\": [0, 70000]\n",
+ " \"bins\": 35000, # number of bins \n",
+ " \"bin_range\": [0, 35000]\n",
" }\n",
"\n",
" # For the numpy histograms on the last cell of the notebook:\n",
" Event_Bin_Dict = {\n",
" \"event_bins\": 1000, # number of bins \n",
- " \"b_range\": [0, 50000] # bin range \n",
+ " \"b_range\": [0, 35000] # bin range \n",
" }\n",
" \n",
"#TODO: make it more adaptive for more than only 2 gains [below was for gain==64 only]\n",
@@ -331,7 +331,7 @@
" # For the numpy histograms on the last cell of the notebook:\n",
" Event_Bin_Dict = {\n",
" \"event_bins\": 1000, # number of bins \n",
- " \"b_range\": [0, 3000] # bin range \n",
+ " \"b_range\": [0, 5000] # bin range \n",
" }\n",
" \n",
"bins = Hist_Bin_Dict[\"bins\"]\n",
@@ -343,12 +343,12 @@
"# of the first peak region are used as cti_limit_low and cti_limit_high:\n",
"\n",
"if gain == 1:\n",
- " cti_limit_low = 3000 # lower limit of cti\n",
- " cti_limit_high = 10000 # higher limit of cti\n",
+ " cti_limit_low = 1000 # lower limit of cti\n",
+ " cti_limit_high = 100000 # higher limit of cti\n",
"#TODO: make it more adaptive for more than only 2 gains [below was for gain==64 only\n",
"else:\n",
" cti_limit_low = 50\n",
- " cti_limit_high = 170"
+ " cti_limit_high = 2000"
]
},
{
@@ -572,22 +572,24 @@
" cores=cpuCores,\n",
" blockSize=blockSize)\n",
" histCalCommonModeCor.debug()\n",
+ " \n",
+ "if corr_bools.get('pattern_class'):\n",
"# Will contain split events pattern data:\n",
- "histCalPcorr = xcal.HistogramCalculator(sensorSize, \n",
- " bins=bins, \n",
- " range=bin_range,\n",
- " nCells=memoryCells, \n",
- " cores=cpuCores,\n",
- " blockSize=blockSize)\n",
- "histCalPcorr.debug()\n",
+ " histCalPcorr = xcal.HistogramCalculator(sensorSize, \n",
+ " bins=bins, \n",
+ " range=bin_range,\n",
+ " nCells=memoryCells, \n",
+ " cores=cpuCores,\n",
+ " blockSize=blockSize)\n",
+ " histCalPcorr.debug()\n",
"# Will contain singles events data:\n",
- "histCalPcorrS = xcal.HistogramCalculator(sensorSize, \n",
- " bins=bins, \n",
- " range=bin_range,\n",
- " nCells=memoryCells, \n",
- " cores=cpuCores,\n",
- " blockSize=blockSize)\n",
- "histCalPcorrS.debug()\n",
+ " histCalPcorrS = xcal.HistogramCalculator(sensorSize, \n",
+ " bins=bins, \n",
+ " range=bin_range,\n",
+ " nCells=memoryCells, \n",
+ " cores=cpuCores,\n",
+ " blockSize=blockSize)\n",
+ " histCalPcorrS.debug()\n",
"if corr_bools.get('relgain'):\n",
" # Will contain gain corrected data:\n",
" histCalGainCor = xcal.HistogramCalculator(sensorSize, \n",
@@ -727,6 +729,22 @@
" offset_mean_im = np.nanmean(data, axis=2) \n",
" offset_single_im = data[...,0] # The offset corrected image corresponding to the first frame \n",
" \n",
+ " # cm: common mode, c: classifications, p: even patterns\n",
+ " if corr_bools.get('common_mode'):\n",
+ " ddsetcm = ofile.create_dataset(h5path+\"/pixels_cm\",\n",
+ " oshape,\n",
+ " chunks=(chunk_size_idim, oshape[1], oshape[2]),\n",
+ " dtype=np.float32)\n",
+ " \n",
+ " data = cmCorrection.correct(data.astype(np.float32), # common mode correction\n",
+ " cellTable=np.zeros(data.shape[2], np.int32)) \n",
+ " histCalCommonModeCor.fill(data) # filling histogram with common mode corrected data\n",
+ " # common mode corrected images:\n",
+ " if cm_mean_im is None:\n",
+ " cm_mean_im = np.nanmean(data, axis=2) \n",
+ " cm_single_im = data[...,0] # The common mode corrected image corresponding to the first frame \n",
+ " ddsetcm[...] = np.moveaxis(data, 2, 0)\n",
+ " \n",
" if corr_bools.get('relgain'):\n",
" data /= rg # relative gain correction \n",
" histCalGainCor.fill(data) # filling histogram with gain corrected data\n",
@@ -739,12 +757,6 @@
"\n",
" if corr_bools.get('pattern_class'):\n",
"\n",
- " # cm: common mode, c: classifications, p: even patterns\n",
- " if corr_bools.get('common_mode'):\n",
- " ddsetcm = ofile.create_dataset(h5path+\"/pixels_cm\",\n",
- " oshape,\n",
- " chunks=(chunk_size_idim, oshape[1], oshape[2]),\n",
- " dtype=np.float32)\n",
"\n",
" ddsetc = ofile.create_dataset(h5path+\"/pixels_classified\",\n",
" oshape,\n",
@@ -759,16 +771,6 @@
" # The calculation of the cluster map:\n",
" patternClassifierLH._noisemap = noise[:, :pixels_x//2, :]\n",
" patternClassifierRH._noisemap = noise[:, pixels_x//2:, :]\n",
- " if corr_bools.get('common_mode'):\n",
- " data = cmCorrection.correct(data.astype(np.float32), # common mode correction\n",
- " cellTable=np.zeros(data.shape[2], np.int32)) \n",
- " histCalCommonModeCor.fill(data) # filling histogram with common mode corrected data\n",
- " \n",
- " if cm_mean_im is None:\n",
- " cm_mean_im = np.nanmean(data, axis=2) \n",
- " cm_single_im = data[...,0] # The common mode corrected image corresponding to the first frame \n",
- " \n",
- " ddsetcm[...] = np.moveaxis(data, 2, 0)\n",
"\n",
" # Dividing the data into left and right hemispheres:\n",
" dataLH = data[:, :pixels_x//2, :]\n",
@@ -825,8 +827,9 @@
" cm_cor_HistVals, _, cm_HistMids, _ = histCalCommonModeCor.get()\n",
"if corr_bools.get('relgain'):\n",
" gain_cor_HistVals, _, gain_cor_HistMids, _ = histCalGainCor.get()\n",
- "split_HistVals, _, split_HistMids, _ = histCalPcorr.get() # split events corrected\n",
- "singles_HistVals, _, singles_HistMids, _ = histCalPcorrS.get() # last s in variable names: singles events"
+ "if corr_bools.get('pattern_class'):\n",
+ " split_HistVals, _, split_HistMids, _ = histCalPcorr.get() # split events corrected\n",
+ " singles_HistVals, _, singles_HistMids, _ = histCalPcorrS.get() # last s in variable names: singles events"
]
},
{
@@ -843,8 +846,9 @@
" np.savez(os.path.join(out_folder, 'Common_Mode_Corrected_Events.npz'), cm_HistMids, cm_cor_HistVals)\n",
"if corr_bools.get('relgain'):\n",
" np.savez(os.path.join(out_folder, 'Gain_Corrected_Events.npz'), gain_cor_HistMids, gain_cor_HistVals)\n",
- "np.savez(os.path.join(out_folder, 'Split_Events.npz'), split_HistMids, split_HistVals)\n",
- "np.savez(os.path.join(out_folder, 'Singles_Events.npz'), singles_HistMids, singles_HistVals)\n",
+ "if corr_bools.get('pattern_class'):\n",
+ " np.savez(os.path.join(out_folder, 'Split_Events.npz'), split_HistMids, split_HistVals)\n",
+ " np.savez(os.path.join(out_folder, 'Singles_Events.npz'), singles_HistMids, singles_HistVals)\n",
"\n",
"print(\"Various spectra are saved to disk in the form of histograms. Please check {}\".format(out_folder))"
]
@@ -859,10 +863,10 @@
"# good.\n",
"\n",
"if gain == 1:\n",
- " x_range = (0, 30000)\n",
+ " x_range = (0, 35000)\n",
"#TODO: make it more adaptive for more than only 2 gains [below was for gain==64 only\n",
"else:\n",
- " x_range = (0, 1000)"
+ " x_range = (0, 2000)"
]
},
{
diff --git a/requirements.txt b/requirements.txt
index b6cb2c479ebd526801df3e03332b7ade04bc9c22..196c198b290133f6166826083d8a409eeb22d424 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-git+file:///gpfs/exfel/sw/calsoft/git/cal_db_interactive@1.5.2
+git+file:///gpfs/exfel/sw/calsoft/git/cal_db_interactive@1.5.3
git+file:///gpfs/exfel/sw/calsoft/git/nbparameterise@0.3
git+file:///gpfs/exfel/sw/calsoft/git/pyDetLib@2.5.3-2.7.0#subdirectory=lib
astcheck == 0.2.5
diff --git a/webservice/sqlite_view.py b/webservice/sqlite_view.py
new file mode 100644
index 0000000000000000000000000000000000000000..854cc3149242b65e3356a8acfdadb787d6748003
--- /dev/null
+++ b/webservice/sqlite_view.py
@@ -0,0 +1,29 @@
+import argparse
+
+import sqlite3
+
+
+parser = argparse.ArgumentParser(
+ description='Update run status at MDC for a given run id.')
+parser.add_argument('--sqlite-fpath', type=str, help='Path to sqlite file path',
+ default='/home/xcal/calibration_webservice/webservice/webservice_jobs.sqlite') # noqa
+parser.add_argument('--run', type=str, help='The run number required '
+ ' for checking its job status.')
+parser.add_argument('--proposal', type=str, help='Proposal numer')
+
+args = vars(parser.parse_args())
+
+sqlite_fpath = args['sqlite_fpath']
+proposal = args['proposal'].zfill(6)
+run = args['run']
+
+conn = sqlite3.connect(sqlite_fpath)
+c = conn.cursor()
+
+c.execute("SELECT * FROM jobs")
+
+for r in c.fetchall():
+ rid, jobid, db_proposal, db_run, status, time, _, _ = r
+ if db_proposal == proposal and db_run == run:
+ print(r)
+
diff --git a/webservice/update_mdc.py b/webservice/update_mdc.py
index 2200972a25a98ff9ba6cce4a48eda1584d63f170..56da834f642f769568fd0fbaf9faf3338ab5d46e 100644
--- a/webservice/update_mdc.py
+++ b/webservice/update_mdc.py
@@ -1,8 +1,10 @@
import argparse
-from metadata_client.metadata_client import MetadataClient
import yaml
+from metadata_client.metadata_client import MetadataClient
+
+
parser = argparse.ArgumentParser(
description='Update run status at MDC for a given run id.')
parser.add_argument('--conf-file', type=str, help='Path to webservice config',
@@ -39,3 +41,4 @@ if response.status_code == 200:
print('Run is updated')
else:
print(f'Update failed {response}')
+
diff --git a/webservice/webservice.py b/webservice/webservice.py
index f8932146aee3fa0ae6631b315dd0c250b9260ed8..5486ecc14d023495223852ebf4303e9bc4257165 100644
--- a/webservice/webservice.py
+++ b/webservice/webservice.py
@@ -706,8 +706,8 @@ async def server_runner(config, mode):
for karabo_id in karabo_ids:
# use selected karabo_das
- karabo_da = data_conf[karabo_id]["karabo-da"] \
- if karabo_das[0] == "all" else karabo_das
+ if karabo_das[0] == 'all':
+ karabo_da = data_conf[karabo_id]["karabo-da"]
# Check if any files for given karabo-das exists
if await check_files(in_folder, wait_runs, karabo_da):
diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py
index 890e02588b758bc28debdcdc7bafff5eb5c75a6b..8bdbdbe15f2e31e43f464ae62d845cc4d71494ea 100755
--- a/xfel_calibrate/calibrate.py
+++ b/xfel_calibrate/calibrate.py
@@ -506,7 +506,7 @@ def create_finalize_script(fmt_args, temp_path, job_list):
tmpl = Template('''
#!/bin/tcsh
source /etc/profile.d/modules.sh
- module load texlive
+ module load texlive/2019
echo 'Running finalize script'
python3 -c "from xfel_calibrate.finalize import finalize;
finalize(joblist={{joblist}},
diff --git a/xfel_calibrate/notebooks.py b/xfel_calibrate/notebooks.py
index fe16bfb27f8130a3fcb4a7568d2e0b6ef53695e8..7c73714ea8f52a6bdf061ae1edfe8417e1a7eaf7 100644
--- a/xfel_calibrate/notebooks.py
+++ b/xfel_calibrate/notebooks.py
@@ -170,7 +170,7 @@ notebooks = {
"notebook":
"notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb", # noqa
"concurrency": {"parameter": "karabo_da",
- "default concurrency": None,
+ "default concurrency": list(range(8)),
"cluster cores": 4},
},
"CORRECT": {