From 9cdfefb73193270dbf8f51757706271b04ac9ef7 Mon Sep 17 00:00:00 2001
From: Steffen Hauf <xcal@max-exfl016.desy.de>
Date: Mon, 31 Aug 2020 13:36:21 +0200
Subject: [PATCH] Deployed version as on 31.08.2020 on xcal
---
bin/activate.sh | 4 +-
.../playground/AGIPD_SingleM_test_Dark.ipynb | 55 +++++++++-
.../pnCCD/Characterize_pnCCD_Dark_NBC.ipynb | 3 +-
notebooks/pnCCD/Correct_pnCCD_NBC.ipynb | 102 +++++++++---------
webservice/sqlite_view.py | 17 +++
webservice/update_mdc.py | 38 +++----
webservice/webservice.py | 4 +-
xfel_calibrate/calibrate.py | 2 +-
xfel_calibrate/notebooks.py | 2 +-
9 files changed, 147 insertions(+), 80 deletions(-)
create mode 100644 webservice/sqlite_view.py
diff --git a/bin/activate.sh b/bin/activate.sh
index 9584c3b78..8c8a0500e 100644
--- a/bin/activate.sh
+++ b/bin/activate.sh
@@ -1,5 +1,5 @@
source /etc/profile.d/modules.sh
module load anaconda/3
-module load texlive
+module load texlive/2019
# export path to python environment
-export PATH=/home/${USER}/.local/bin:$PATH
+export PATH=$HOME/.local/bin:$PATH
diff --git a/notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb b/notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb
index b0d22ac4e..ec7f92ae5 100644
--- a/notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb
+++ b/notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb
@@ -740,10 +740,8 @@
]
},
{
- "cell_type": "code",
- "execution_count": null,
+ "cell_type": "markdown",
"metadata": {},
- "outputs": [],
"source": [
"table = []\n",
"gain_names = ['High', 'Medium', 'Low']\n",
@@ -858,7 +856,56 @@
"execution_count": null,
"metadata": {},
"outputs": [],
- "source": []
+ "source": [
+ "import tabulate\n",
+ "\n",
+ "header = ['Parameter', \n",
+ " \"New constant\", \"Old constant \", \n",
+ " \"New constant\", \"Old constant \", \n",
+ " \"New constant\", \"Old constant \",\n",
+ " \"New constant\", \"Old constant \"]\n",
+ "\n",
+ "for const in ['Offset', 'Noise', 'ThresholdsDark']:\n",
+ " if const != 'ThresholdsDark':\n",
+ " table = [['','High gain', 'High gain', 'Medium gain', 'Medium gain', 'Low gain', 'Low gain']]\n",
+ " else:\n",
+ " table = [['','HG-MG threshold', 'HG-MG threshold', 'MG-LG threshold', 'MG-LG threshold']]\n",
+ "\n",
+ " data = np.copy(res[const])\n",
+ " if const == 'ThresholdsDark':\n",
+ " data[...,0][res['BadPixelsDark'][...,0]>0] = np.nan\n",
+ " data[...,1][res['BadPixelsDark'][...,1]>0] = np.nan\n",
+ " else:\n",
+ " data[res['BadPixelsDark']>0] = np.nan\n",
+ "\n",
+ " if old_const[const] is not None and old_const['BadPixelsDark'] is not None:\n",
+ " dataold = np.copy(old_const[const])\n",
+ " if const == 'ThresholdsDark':\n",
+ " dataold[...,0][old_const['BadPixelsDark'][...,0]>0] = np.nan\n",
+ " dataold[...,1][old_const['BadPixelsDark'][...,1]>0] = np.nan\n",
+ " else:\n",
+ " dataold[old_const['BadPixelsDark']>0] = np.nan\n",
+ "\n",
+ " f_list = [np.nanmedian, np.nanmean, np.nanstd, np.nanmin, np.nanmax]\n",
+ " n_list = ['Median', 'Mean', 'Std', 'Min', 'Max']\n",
+ "\n",
+ " for i, f in enumerate(f_list):\n",
+ " line = [n_list[i]]\n",
+ " for gain in range(3):\n",
+ " # Compare only 3 threshold gain-maps\n",
+ " if gain == 2 and const == 'ThresholdsDark':\n",
+ " continue\n",
+ " line.append('{:6.1f}'.format(f(data[...,gain])))\n",
+ " if old_const[const] is not None and old_const['BadPixelsDark'] is not None:\n",
+ " line.append('{:6.1f}'.format(f(dataold[...,gain])))\n",
+ " else:\n",
+ " line.append('-')\n",
+ "\n",
+ " table.append(line)\n",
+ "\n",
+ " display(Markdown('### {} [ADU], good pixels only ###'.format(const)))\n",
+ " md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=header))) "
+ ]
}
],
"metadata": {
diff --git a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
index d01308896..ec232229b 100644
--- a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
+++ b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
@@ -56,7 +56,7 @@
"number_dark_frames = 0 # number of images to be used, if set to 0 all available images are used\n",
"chunkSize = 100 # number of images to read per chunk\n",
"fix_temperature = 0. # fix temperature in K, set to 0. to use value from slow data\n",
- "gain = 0. # the detector's gain setting, It is later read from file and this value is overwritten\n",
+ "gain = 1 # the detector's gain setting, It is later read from file and this value is overwritten\n",
"bias_voltage = 0. # the detector's bias voltage. set to 0. to use value from slow data.\n",
"integration_time = 70 # detector's integration time\n",
"commonModeAxis = 0 # axis along which common mode will be calculated (0: along rows, 1: along columns)\n",
@@ -206,7 +206,6 @@
" try:\n",
" with h5py.File(ctrl_fname, \"r\") as f:\n",
" if bias_voltage == 0.:\n",
- " print(\"bias voltage control h5path:\", os.path.join(mdl_ctrl_path, \"DAQ_MPOD/u0voltage/value\"))\n",
" bias_voltage = abs(f[os.path.join(mdl_ctrl_path, \"DAQ_MPOD/u0voltage/value\")][0])\n",
" gain = f[os.path.join(mdl_ctrl_path, \"DAQ_GAIN/pNCCDGain/value\")][0]\n",
" if fix_temperature == 0.:\n",
diff --git a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
index b1484e339..18c45d5bd 100644
--- a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
+++ b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
@@ -22,10 +22,10 @@
},
"outputs": [],
"source": [
- "in_folder = \"/gpfs/exfel/exp/SQS/202031/p900166/raw\" # input folder\n",
- "out_folder = '/gpfs/exfel/data/scratch/ahmedk/test/pnccd' # output folder\n",
- "run = 39 # which run to read data from\n",
- "sequences = [0] # sequences to correct, set to -1 for all, range allowed\n",
+ "in_folder = \"/gpfs/exfel/exp/SQS/202022/p002720/raw\" # input folder\n",
+ "out_folder = '/gpfs/exfel/data/scratch/setoodeh' # output folder\n",
+ "run = 53 # which run to read data from\n",
+ "sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"\n",
"db_module = \"pnCCD_M205_M206\"\n",
"karabo_da = 'PNCCD01' # data aggregators\n",
@@ -58,7 +58,7 @@
"gain = 0. # the detector's gain setting, It is later read from file and this value is overwritten\n",
"bias_voltage = 0. # the detector's bias voltage. set to 0. to use value from slow data.\n",
"integration_time = 70\n",
- "photon_energy = 1.6 # Al fluorescence in keV\n",
+ "photon_energy = 1.5 # Al fluorescence in keV\n",
"\n",
"cal_db_interface = \"tcp://max-exfl016:8015\" # calibration DB interface to use\n",
"cal_db_timeout = 300000 # timeout on caldb requests\n",
@@ -68,7 +68,7 @@
"common_mode = True # Apply common mode correction\n",
"relgain = True # Apply relative gain correction\n",
"cti = False # Apply charge transfer inefficiency correction (not implemented, yet)\n",
- "do_pattern_classification = True # classify split events"
+ "do_pattern_classification = False # classify split events"
]
},
{
@@ -311,14 +311,14 @@
"# For all xcal histograms:\n",
"if gain == 1:\n",
" Hist_Bin_Dict = {\n",
- " \"bins\": 70000, # number of bins \n",
- " \"bin_range\": [0, 70000]\n",
+ " \"bins\": 35000, # number of bins \n",
+ " \"bin_range\": [0, 35000]\n",
" }\n",
"\n",
" # For the numpy histograms on the last cell of the notebook:\n",
" Event_Bin_Dict = {\n",
" \"event_bins\": 1000, # number of bins \n",
- " \"b_range\": [0, 50000] # bin range \n",
+ " \"b_range\": [0, 35000] # bin range \n",
" }\n",
" \n",
"#TODO: make it more adaptive for more than only 2 gains [below was for gain==64 only]\n",
@@ -331,7 +331,7 @@
" # For the numpy histograms on the last cell of the notebook:\n",
" Event_Bin_Dict = {\n",
" \"event_bins\": 1000, # number of bins \n",
- " \"b_range\": [0, 3000] # bin range \n",
+ " \"b_range\": [0, 5000] # bin range \n",
" }\n",
" \n",
"bins = Hist_Bin_Dict[\"bins\"]\n",
@@ -343,12 +343,12 @@
"# of the first peak region are used as cti_limit_low and cti_limit_high:\n",
"\n",
"if gain == 1:\n",
- " cti_limit_low = 3000 # lower limit of cti\n",
- " cti_limit_high = 10000 # higher limit of cti\n",
+ " cti_limit_low = 1000 # lower limit of cti\n",
+ " cti_limit_high = 100000 # higher limit of cti\n",
"#TODO: make it more adaptive for more than only 2 gains [below was for gain==64 only\n",
"else:\n",
" cti_limit_low = 50\n",
- " cti_limit_high = 170"
+ " cti_limit_high = 2000"
]
},
{
@@ -572,22 +572,24 @@
" cores=cpuCores,\n",
" blockSize=blockSize)\n",
" histCalCommonModeCor.debug()\n",
+ " \n",
+ "if corr_bools.get('pattern_class'):\n",
"# Will contain split events pattern data:\n",
- "histCalPcorr = xcal.HistogramCalculator(sensorSize, \n",
- " bins=bins, \n",
- " range=bin_range,\n",
- " nCells=memoryCells, \n",
- " cores=cpuCores,\n",
- " blockSize=blockSize)\n",
- "histCalPcorr.debug()\n",
+ " histCalPcorr = xcal.HistogramCalculator(sensorSize, \n",
+ " bins=bins, \n",
+ " range=bin_range,\n",
+ " nCells=memoryCells, \n",
+ " cores=cpuCores,\n",
+ " blockSize=blockSize)\n",
+ " histCalPcorr.debug()\n",
"# Will contain singles events data:\n",
- "histCalPcorrS = xcal.HistogramCalculator(sensorSize, \n",
- " bins=bins, \n",
- " range=bin_range,\n",
- " nCells=memoryCells, \n",
- " cores=cpuCores,\n",
- " blockSize=blockSize)\n",
- "histCalPcorrS.debug()\n",
+ " histCalPcorrS = xcal.HistogramCalculator(sensorSize, \n",
+ " bins=bins, \n",
+ " range=bin_range,\n",
+ " nCells=memoryCells, \n",
+ " cores=cpuCores,\n",
+ " blockSize=blockSize)\n",
+ " histCalPcorrS.debug()\n",
"if corr_bools.get('relgain'):\n",
" # Will contain gain corrected data:\n",
" histCalGainCor = xcal.HistogramCalculator(sensorSize, \n",
@@ -727,6 +729,22 @@
" offset_mean_im = np.nanmean(data, axis=2) \n",
" offset_single_im = data[...,0] # The offset corrected image corresponding to the first frame \n",
" \n",
+ " # cm: common mode, c: classifications, p: even patterns\n",
+ " if corr_bools.get('common_mode'):\n",
+ " ddsetcm = ofile.create_dataset(h5path+\"/pixels_cm\",\n",
+ " oshape,\n",
+ " chunks=(chunk_size_idim, oshape[1], oshape[2]),\n",
+ " dtype=np.float32)\n",
+ " \n",
+ " data = cmCorrection.correct(data.astype(np.float32), # common mode correction\n",
+ " cellTable=np.zeros(data.shape[2], np.int32)) \n",
+ " histCalCommonModeCor.fill(data) # filling histogram with common mode corrected data\n",
+ " # common mode corrected images:\n",
+ " if cm_mean_im is None:\n",
+ " cm_mean_im = np.nanmean(data, axis=2) \n",
+ " cm_single_im = data[...,0] # The common mode corrected image corresponding to the first frame \n",
+ " ddsetcm[...] = np.moveaxis(data, 2, 0)\n",
+ " \n",
" if corr_bools.get('relgain'):\n",
" data /= rg # relative gain correction \n",
" histCalGainCor.fill(data) # filling histogram with gain corrected data\n",
@@ -739,12 +757,6 @@
"\n",
" if corr_bools.get('pattern_class'):\n",
"\n",
- " # cm: common mode, c: classifications, p: even patterns\n",
- " if corr_bools.get('common_mode'):\n",
- " ddsetcm = ofile.create_dataset(h5path+\"/pixels_cm\",\n",
- " oshape,\n",
- " chunks=(chunk_size_idim, oshape[1], oshape[2]),\n",
- " dtype=np.float32)\n",
"\n",
" ddsetc = ofile.create_dataset(h5path+\"/pixels_classified\",\n",
" oshape,\n",
@@ -759,16 +771,6 @@
" # The calculation of the cluster map:\n",
" patternClassifierLH._noisemap = noise[:, :pixels_x//2, :]\n",
" patternClassifierRH._noisemap = noise[:, pixels_x//2:, :]\n",
- " if corr_bools.get('common_mode'):\n",
- " data = cmCorrection.correct(data.astype(np.float32), # common mode correction\n",
- " cellTable=np.zeros(data.shape[2], np.int32)) \n",
- " histCalCommonModeCor.fill(data) # filling histogram with common mode corrected data\n",
- " \n",
- " if cm_mean_im is None:\n",
- " cm_mean_im = np.nanmean(data, axis=2) \n",
- " cm_single_im = data[...,0] # The common mode corrected image corresponding to the first frame \n",
- " \n",
- " ddsetcm[...] = np.moveaxis(data, 2, 0)\n",
"\n",
" # Dividing the data into left and right hemispheres:\n",
" dataLH = data[:, :pixels_x//2, :]\n",
@@ -825,8 +827,9 @@
" cm_cor_HistVals, _, cm_HistMids, _ = histCalCommonModeCor.get()\n",
"if corr_bools.get('relgain'):\n",
" gain_cor_HistVals, _, gain_cor_HistMids, _ = histCalGainCor.get()\n",
- "split_HistVals, _, split_HistMids, _ = histCalPcorr.get() # split events corrected\n",
- "singles_HistVals, _, singles_HistMids, _ = histCalPcorrS.get() # last s in variable names: singles events"
+ "if corr_bools.get('pattern_class'):\n",
+ " split_HistVals, _, split_HistMids, _ = histCalPcorr.get() # split events corrected\n",
+ " singles_HistVals, _, singles_HistMids, _ = histCalPcorrS.get() # last s in variable names: singles events"
]
},
{
@@ -843,8 +846,9 @@
" np.savez(os.path.join(out_folder, 'Common_Mode_Corrected_Events.npz'), cm_HistMids, cm_cor_HistVals)\n",
"if corr_bools.get('relgain'):\n",
" np.savez(os.path.join(out_folder, 'Gain_Corrected_Events.npz'), gain_cor_HistMids, gain_cor_HistVals)\n",
- "np.savez(os.path.join(out_folder, 'Split_Events.npz'), split_HistMids, split_HistVals)\n",
- "np.savez(os.path.join(out_folder, 'Singles_Events.npz'), singles_HistMids, singles_HistVals)\n",
+ "if corr_bools.get('pattern_class'):\n",
+ " np.savez(os.path.join(out_folder, 'Split_Events.npz'), split_HistMids, split_HistVals)\n",
+ " np.savez(os.path.join(out_folder, 'Singles_Events.npz'), singles_HistMids, singles_HistVals)\n",
"\n",
"print(\"Various spectra are saved to disk in the form of histograms. Please check {}\".format(out_folder))"
]
@@ -859,10 +863,10 @@
"# good.\n",
"\n",
"if gain == 1:\n",
- " x_range = (0, 30000)\n",
+ " x_range = (0, 35000)\n",
"#TODO: make it more adaptive for more than only 2 gains [below was for gain==64 only\n",
"else:\n",
- " x_range = (0, 1000)"
+ " x_range = (0, 2000)"
]
},
{
diff --git a/webservice/sqlite_view.py b/webservice/sqlite_view.py
new file mode 100644
index 000000000..2979f2223
--- /dev/null
+++ b/webservice/sqlite_view.py
@@ -0,0 +1,17 @@
+import sqlite3
+
+
+file_path = "/home/xcal/calibration_webservice/webservice/webservice_jobs.sqlite"
+run = '247'
+proposal = '900138'
+
+
+conn = sqlite3.connect(file_path)
+c = conn.cursor()
+
+c.execute("SELECT * FROM jobs")
+
+for r in c.fetchall():
+ rid, jobid, db_proposal, db_run, status, time, _, _ = r
+ if db_proposal == proposal and db_run == run:
+ print(r)
diff --git a/webservice/update_mdc.py b/webservice/update_mdc.py
index 2200972a2..36eb34b93 100644
--- a/webservice/update_mdc.py
+++ b/webservice/update_mdc.py
@@ -1,17 +1,17 @@
+import yaml
import argparse
from metadata_client.metadata_client import MetadataClient
-import yaml
+
parser = argparse.ArgumentParser(
- description='Update run status at MDC for a given run id.')
-parser.add_argument('--conf-file', type=str, help='Path to webservice config',
- default='/home/xcal/calibration_webservice_deployed/webservice/webservice.yaml') # noqa
-parser.add_argument('--flg', type=str, choices=["NA", "R", "A"],
- help='Status flag for MDC request: NA - not available, R - running, A - available.') # noqa
+ description='Request dark characterization. Will wait on data transfers to complete first!')
+parser.add_argument('--conf-file', type=str, help='Path to webcervice config', default='/home/xcal/calibration_webservice_deployed/webservice/webservice.yaml')
+parser.add_argument('--flg', type=str, choices=["NA", "R", "A"], help='Status flag for MDC request')
parser.add_argument('--rid', type=int, help='Run id from MDC')
-parser.add_argument('--msg', type=str, help='Message string to MDC',
- default='Error while job submission')
+parser.add_argument('--msg', type=str, help='Message string to MDC')
+
+
args = vars(parser.parse_args())
conf_file = args['conf_file']
@@ -19,23 +19,23 @@ rid = args['rid']
flg = args['flg']
msg = args['msg']
+
with open(conf_file, "r") as f:
config = yaml.load(f.read(), Loader=yaml.FullLoader)
mdconf = config['metadata-client']
client_conn = MetadataClient(client_id=mdconf['user-id'],
- client_secret=mdconf['user-secret'],
- user_email=mdconf['user-email'],
- token_url=mdconf['token-url'],
- refresh_url=mdconf['refresh-url'],
- auth_url=mdconf['auth-url'],
- scope=mdconf['scope'],
- base_api_url=mdconf['base-api-url'])
+ client_secret=mdconf['user-secret'],
+ user_email=mdconf['user-email'],
+ token_url=mdconf['token-url'],
+ refresh_url=mdconf['refresh-url'],
+ auth_url=mdconf['auth-url'],
+ scope=mdconf['scope'],
+ base_api_url=mdconf['base-api-url'])
+
response = client_conn.update_run_api(rid, {'flg_cal_data_status': flg,
'cal_pipeline_reply': msg})
-if response.status_code == 200:
- print('Run is updated')
-else:
- print(f'Update failed {response}')
+print(response, response.status_code)
+
diff --git a/webservice/webservice.py b/webservice/webservice.py
index f8932146a..5486ecc14 100644
--- a/webservice/webservice.py
+++ b/webservice/webservice.py
@@ -706,8 +706,8 @@ async def server_runner(config, mode):
for karabo_id in karabo_ids:
# use selected karabo_das
- karabo_da = data_conf[karabo_id]["karabo-da"] \
- if karabo_das[0] == "all" else karabo_das
+ if karabo_das[0] == 'all':
+ karabo_da = data_conf[karabo_id]["karabo-da"]
# Check if any files for given karabo-das exists
if await check_files(in_folder, wait_runs, karabo_da):
diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py
index 890e02588..8bdbdbe15 100755
--- a/xfel_calibrate/calibrate.py
+++ b/xfel_calibrate/calibrate.py
@@ -506,7 +506,7 @@ def create_finalize_script(fmt_args, temp_path, job_list):
tmpl = Template('''
#!/bin/tcsh
source /etc/profile.d/modules.sh
- module load texlive
+ module load texlive/2019
echo 'Running finalize script'
python3 -c "from xfel_calibrate.finalize import finalize;
finalize(joblist={{joblist}},
diff --git a/xfel_calibrate/notebooks.py b/xfel_calibrate/notebooks.py
index fe16bfb27..5fb7ff7b3 100644
--- a/xfel_calibrate/notebooks.py
+++ b/xfel_calibrate/notebooks.py
@@ -170,7 +170,7 @@ notebooks = {
"notebook":
"notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb", # noqa
"concurrency": {"parameter": "karabo_da",
- "default concurrency": None,
+ "default concurrency": list(range(6)),
"cluster cores": 4},
},
"CORRECT": {
--
GitLab