diff --git a/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb b/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb
index 19979412413f46cd920d18ed024cf03f581b297c..02b8a3a3f807b0927b076364b78dcd78d2a8d698 100644
--- a/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb
+++ b/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb
@@ -22,22 +22,20 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
"start_date = \"2018-01-30\" # Date to start investigation interval from\n",
"end_date = \"2018-12-12\" # Date to end investigation interval at, can be \"now\"\n",
"constants = [\"Noise\", \"SlopesFF\", \"SlopesPC\", \"Offset\"] # Constants to plot\n",
- "modules = [0] # Modules, range allowed\n",
+ "modules = [1] # Modules, range allowed\n",
"bias_voltages = [300, 500] # Bias voltage\n",
"mem_cells = [128, 176] # Number of used memory cells. Typically: 4,32,64,128,176.\n",
"photon_energy = 9.2 # Photon energy of the beam\n",
- "out_folder = \"/gpfs/exfel/data/scratch/karnem/testAGIPD16_10/\" # Output folder, required\n",
- "use_existing = \"/gpfs/exfel/data/scratch/karnem/testAGIPD16_10/\" # Input folder\n",
- "cal_db_timeout = 180000 # timeout on caldb requests\",\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/karnem/testAGIPD16_21/\" # Output folder, required\n",
+ "use_existing = \"\" # Input folder\n",
+ "cal_db_timeout = 120000 # timeout on caldb requests\",\n",
"adu_to_photon = 33.17 # ADU to photon conversion factor (8000 / 3.6 / 67.)\n",
"nMemToShow = 32 # Number of memory cells to be shown in plots over ASICs\n",
"db_module = \"AGIPD1M1\" # detector entry in the DB to investigate\n",
@@ -55,11 +53,11 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
"scrolled": true
},
"outputs": [],
"source": [
+ "import copy\n",
"import datetime\n",
"import dateutil.parser\n",
"import numpy as np\n",
@@ -68,10 +66,11 @@
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"\n",
+ "import h5py\n",
"import matplotlib\n",
- "% matplotlib inline\n",
+ "%matplotlib inline\n",
"\n",
- "from iCalibrationDB import Constants, Conditions, Detectors\n",
+ "from iCalibrationDB import Constants, Conditions, Detectors, ConstantMetaData\n",
"from cal_tools.tools import get_from_db, get_random_db_interface\n",
"from cal_tools.ana_tools import (save_dict_to_hdf5, load_data_from_hdf5, \n",
" combine_constants, HMType,\n",
@@ -81,14 +80,10 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Prepare variables\n",
- "interval = 1 # interval for evaluation in days\n",
- "\n",
"nMem = max(mem_cells) # Number of mem Cells to store\n",
"spShape = (64,64) # Shape of superpixel\n",
"\n",
@@ -102,13 +97,14 @@
" 'Offset': 'BadPixelsDark'}\n",
"print('Bad pixels data: ', constantsDark)\n",
"\n",
+ "base_path = '/gpfs/exfel/d/cal/caldb_store/'\n",
+ "print('Base file path: {}'.format(base_path))\n",
+ "\n",
"# Define parameters in order to perform loop over time stamps\n",
"start = datetime.datetime.now() if start_date.upper() == \"NOW\" else dateutil.parser.parse(\n",
" start_date)\n",
"end = datetime.datetime.now() if end_date.upper() == \"NOW\" else dateutil.parser.parse(\n",
" end_date)\n",
- "step = datetime.timedelta(hours=interval)\n",
- "dt = end\n",
"\n",
"# Create output folder\n",
"os.makedirs(out_folder, exist_ok=True)\n",
@@ -117,17 +113,165 @@
"det = getattr(Detectors, db_module)\n",
"dconstants = getattr(Constants, dclass)\n",
"\n",
- "cal_db_interface = get_random_db_interface(cal_db_interface)\n",
- "print('CalDB Interface {}'.format(cal_db_interface))"
+ "print('CalDB Interface: {}'.format(cal_db_interface))\n",
+ "print('Start time at: ', start)\n",
+ "print('End time at: ', end)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def list_runner(*args, **kwargs):\n",
+ " params = list(map(tuple, args)) * kwargs.get('repeat', 1)\n",
+ " possible_params = [[]]\n",
+ " for param in params:\n",
+ " possible_params = [x+[y] for x in possible_params for y in param]\n",
+ " \n",
+ " names = kwargs.get('names')\n",
+ " d_possible_params = []\n",
+ " for par in possible_params:\n",
+ " d_possible_params.append(dict(zip(names, par)))\n",
+ "\n",
+ " return d_possible_params\n",
+ "\n",
+ "\n",
+ "parameter_list = list_runner(bias_voltages, modules, mem_cells, names = ['bias_voltage', 'module', 'mem_cells'])\n",
+ "print(parameter_list)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false
+ "scrolled": false
},
"outputs": [],
+ "source": [
+ "# Retrieve list of meta-data\n",
+ "constant_versions = []\n",
+ "constant_parameters = []\n",
+ "constantBP_versions = []\n",
+ "metadata = ConstantMetaData()\n",
+ "\n",
+ "# Loop over constants\n",
+ "for c, const in enumerate(constants):\n",
+ " \n",
+ " if use_existing != \"\":\n",
+ " break\n",
+ " \n",
+ " # Loop over parameters\n",
+ " for pars in parameter_list:\n",
+ "\n",
+ " qm = \"Q{}M{}\".format(pars['module'] // 4 + 1, pars['module'] % 4 + 1)\n",
+ " \n",
+ " if (const in [\"Offset\", \"Noise\", \"SlopesPC\"] or \"DARK\" in const.upper()):\n",
+ " dcond = Conditions.Dark\n",
+ " mcond = getattr(dcond, dclass)(\n",
+ " memory_cells=pars['mem_cells'],\n",
+ " bias_voltage=pars['bias_voltage'])\n",
+ " else:\n",
+ " dcond = Conditions.Illuminated\n",
+ " mcond = getattr(dcond, dclass)(\n",
+ " memory_cells=pars['mem_cells'],\n",
+ " bias_voltage=pars['bias_voltage'],\n",
+ " photon_energy=photon_energy)\n",
+ "\n",
+ " print('Request: ', const, 'with paramters:', pars)\n",
+ " # Request Constant versions for given parameters and module\n",
+ " data = get_from_db(getattr(det, qm),\n",
+ " getattr(dconstants,\n",
+ " const)(),\n",
+ " copy.deepcopy(mcond), None,\n",
+ " cal_db_interface,\n",
+ " creation_time=start,\n",
+ " verbosity=0,\n",
+ " timeout=cal_db_timeout,\n",
+ " meta_only=True,\n",
+ " version_info=True)\n",
+ "\n",
+ " # Request BP constant versions\n",
+ " print('constantDark:', constantsDark[const], ) \n",
+ " dataBP = get_from_db(getattr(det, qm),\n",
+ " getattr(dconstants, \n",
+ " constantsDark[const])(),\n",
+ " copy.deepcopy(mcond), None,\n",
+ " cal_db_interface,\n",
+ " creation_time=start,\n",
+ " verbosity=0,\n",
+ " timeout=cal_db_timeout,\n",
+ " meta_only=True,\n",
+ " version_info=True)\n",
+ " \n",
+ " if not isinstance(data, list) or not isinstance(dataBP, list):\n",
+ " continue\n",
+ " \n",
+ " found_BPmatch = False\n",
+ " for d in data:\n",
+ " # Match prober BP constant version\n",
+ " # and get constant version within\n",
+ " # requested time range\n",
+ " if d is None:\n",
+ " print('Time or data is not found!')\n",
+ " continue\n",
+ "\n",
+ " dt = dateutil.parser.parse(d['begin_at'])\n",
+ "\n",
+ " if dt.replace(tzinfo=None) > end or dt.replace(tzinfo=None) < start:\n",
+ " continue\n",
+ " \n",
+ " max_time = 15 #The max margin in min. for the matching closest BP\n",
+ " closest_BP = None\n",
+ " closest_BPtime = None\n",
+ " \n",
+ " for dBP in dataBP:\n",
+ " if dBP is None:\n",
+ " print(\"Bad pixels are not found!\")\n",
+ " continue\n",
+ " \n",
+ " dt = dateutil.parser.parse(d['begin_at'])\n",
+ " dBPt = dateutil.parser.parse(dBP['begin_at'])\n",
+ " \n",
+ " if dt == dBPt:\n",
+ " found_BPmatch = True\n",
+ " else:\n",
+ "\n",
+ " if np.abs(dBPt-dt).seconds < (max_time*60):\n",
+ " if closest_BP is None:\n",
+ " closest_BP = dBP\n",
+ " closest_BPtime = dBPt\n",
+ " else:\n",
+ " if np.abs(dBPt-dt) < np.abs(closest_BPtime-dt):\n",
+ " closest_BP = dBP\n",
+ " closest_BPtime = dBPt\n",
+ " \n",
+ " if dataBP.index(dBP) == len(dataBP)-1:\n",
+ " if closest_BP:\n",
+ " dBP = closest_BP\n",
+ " dBPt = closest_BPtime\n",
+ " found_BPmatch = True\n",
+ " else:\n",
+ " print('Bad pixels are not found!')\n",
+ " \n",
+ " if found_BPmatch:\n",
+ " print(\"Found constant {}: begin at {}\".format(const, dt))\n",
+ " print(\"Found bad pixels at {}\".format(dBPt))\n",
+ " constantBP_versions.append(dBP)\n",
+ " constant_versions.append(d)\n",
+ " constant_parameters.append(copy.deepcopy(pars))\n",
+ " found_BPmatch = False\n",
+ " break\n",
+ " \n",
+ "print('Number of retrieved constants with a bad pixel match is {}'.format(len(constant_versions)))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
"source": [
"import copy\n",
"\n",
@@ -170,155 +314,82 @@
" \n",
" if len(data.shape) < 4:\n",
" print(data.shape, \"Unexpected shape!\")\n",
- "\n",
" return data\n",
"\n",
+ "\n",
+ "\n",
"ret_constants = {}\n",
+ "for i, constant_version in enumerate(constant_versions):\n",
"\n",
- "if use_existing != '':\n",
- " mem_cells = []\n",
- "\n",
- "# Loop over max_memory cells\n",
- "for the_mem_cells in mem_cells:\n",
- "\n",
- " # Loop over bias voltages\n",
- " for bias_voltage in bias_voltages:\n",
- "\n",
- " # Loop over constants\n",
- " for c, const in enumerate(constants):\n",
- "\n",
- " if not const in ret_constants:\n",
- " ret_constants[const] = {}\n",
- "\n",
- " # Loop over modules\n",
- " for module in modules:\n",
- "\n",
- " dt = end\n",
- "\n",
- " qm = \"Q{}M{}\".format(module // 4 + 1, module % 4 + 1)\n",
- " if not qm in ret_constants[const]:\n",
- " ret_constants[const][qm] = []\n",
- "\n",
- " # Loop over time stamps\n",
- " while dt > start:\n",
- " creation_time = dt\n",
- "\n",
- " if (const in [\"Offset\", \"Noise\",\n",
- " \"SlopesPC\"] or \"DARK\" in const.upper()):\n",
- " dcond = Conditions.Dark\n",
- " mcond = getattr(dcond, dclass)(\n",
- " memory_cells=the_mem_cells,\n",
- " bias_voltage=bias_voltage)\n",
- " else:\n",
- " dcond = Conditions.Illuminated\n",
- " mcond = getattr(dcond, dclass)(\n",
- " memory_cells=the_mem_cells,\n",
- " bias_voltage=bias_voltage,\n",
- " photon_energy=photon_energy)\n",
- "\n",
- " print('Request: ', const, qm, the_mem_cells, bias_voltage,\n",
- " creation_time)\n",
- " cdata, ctime = get_from_db(getattr(det, qm),\n",
- " getattr(dconstants, const)(),\n",
- " copy.deepcopy(mcond),\n",
- " None,\n",
- " cal_db_interface,\n",
- " creation_time=creation_time,\n",
- " verbosity=0,\n",
- " timeout=cal_db_timeout,\n",
- " meta_only=True )\n",
- " \n",
- " if ctime is None or cdata is None:\n",
- " print('Time or Data is None')\n",
- " break\n",
- "\n",
- " ctime = ctime.calibration_constant_version.begin_at\n",
- "\n",
- " print(\"Found constant {}: begin at {}\".format(const,\n",
- " cdata is not None),\n",
- " ctime)\n",
- " print(cdata.shape)\n",
- " cdata = modify_const(const, cdata)\n",
- " print(cdata.shape)\n",
- "\n",
- " # Request bad pixel mask\n",
- " if const in constantsDark:\n",
- " print('Request bad pixels ',\n",
- " (creation_time + step + step + step + step))\n",
- " cdataBP, ctimeBP = get_from_db(getattr(det, qm),\n",
- " getattr(dconstants,\n",
- " constantsDark[const])(),\n",
- " copy.deepcopy(mcond),\n",
- " None,\n",
- " cal_db_interface,\n",
- " creation_time=(\n",
- " creation_time + step + step + step + step),\n",
- " verbosity=0,\n",
- " timeout=cal_db_timeout,\n",
- " meta_only=True)\n",
- " ctimeBP = ctimeBP.calibration_constant_version.begin_at\n",
- "\n",
- " if cdataBP is None:\n",
- " print(\"Bad pixels are not found!\")\n",
- " ctime = ctime.replace(tzinfo=None)\n",
- " dt = ctime - step\n",
- " continue\n",
- "\n",
- " print(\"Found bad pixels {}\".format(ctimeBP))\n",
- " print(cdataBP.shape)\n",
- " cdataBP = modify_const(const, cdataBP, True)\n",
- " print(cdataBP.shape)\n",
- " \n",
- " if cdataBP.shape != cdata.shape:\n",
- " print('Wrong bad pixel shape!')\n",
- " ctime = ctime.replace(tzinfo=None)\n",
- " dt = ctime - step\n",
- " continue\n",
- "\n",
- " # Apply bad pixel mask\n",
- " cdataABP = np.copy(cdata)\n",
- " cdataABP[cdataBP > 0] = np.nan\n",
- "\n",
- " # Create superpixels for constants with BP applied\n",
- " cdataABP = get_rebined(cdataABP, spShape)\n",
- " toStoreBP = prepare_to_store(np.nanmean(cdataABP, axis=(1, 3)), nMem)\n",
- " toStoreBPStd = prepare_to_store(np.nanstd(cdataABP, axis=(1, 3)), nMem)\n",
- " \n",
- " # Prepare number of bad pixels per superpixels\n",
- " cdataBP = get_rebined(cdataBP, spShape)\n",
- " cdataNBP = prepare_to_store(np.nansum(cdataBP > 0, axis=(1, 3)), nMem)\n",
- " else:\n",
- " toStoreExtBP = 0\n",
- " cdataBPExt = 0\n",
- "\n",
- " # Create superpixels for constants without BP applied\n",
- " cdata = get_rebined(cdata, spShape)\n",
- " toStoreStd = prepare_to_store(np.nanstd(cdata, axis=(1, 3)), nMem)\n",
- " toStore = prepare_to_store(np.nanmean(cdata, axis=(1, 3)), nMem)\n",
- "\n",
- " # Convert parameters to dict\n",
- " dpar = {}\n",
- " for par in mcond.parameters:\n",
- " dpar[par.name] = par.value\n",
- "\n",
- " print(\"Store values in dict\", const, qm, ctime)\n",
- " ret_constants[const][qm].append({'ctime': ctime,\n",
- " 'nBP': cdataNBP,\n",
- " 'dataBP': toStoreBP,\n",
- " 'dataBPStd': toStoreBPStd,\n",
- " 'data': toStore,\n",
- " 'dataStd': toStoreStd,\n",
- " 'mdata': dpar})\n",
- " \n",
- " ctime = ctime.replace(tzinfo=None)\n",
- " dt = ctime - step"
+ " const = constant_version['data_set_name'].split('/')[-2]\n",
+ " ctime = dateutil.parser.parse(constant_version['begin_at']).replace(tzinfo=None)\n",
+ " qm = \"Q{}M{}\".format(constant_parameters[i]['module'] // 4 + 1, \n",
+ " constant_parameters[i]['module'] % 4 + 1)\n",
+ " \n",
+ " \n",
+ " print(\"constant: {}, module {}\".format(const,qm))\n",
+ " \n",
+ " if not const in ret_constants:\n",
+ " ret_constants[const] = {}\n",
+ " if not qm in ret_constants[const]:\n",
+ " ret_constants[const][qm] = []\n",
+ " \n",
+ " fpath = \"{}/{}/{}\".format(base_path,\n",
+ " constant_version['path_to_file'],\n",
+ " constant_version['file_name'])\n",
+ " \n",
+ " with h5py.File(fpath, \"r\") as f:\n",
+ " cdata = f[\"{}/data\".format(constant_version['data_set_name'])][()]\n",
+ " \n",
+ "\n",
+ " fpathBP = \"{}/{}/{}\".format(base_path,\n",
+ " constantBP_versions[i]['path_to_file'],\n",
+ " constantBP_versions[i]['file_name'])\n",
+ " \n",
+ " with h5py.File(fpathBP, \"r\") as f:\n",
+ " cdataBP = f[\"{}/data\".format(constantBP_versions[i]['data_set_name'])][()]\n",
+ "\n",
+ " print('constant path: {}\\n BP path: {}'.format(fpath, fpathBP))\n",
+ " \n",
+ " cdata = modify_const(const, cdata)\n",
+ " cdataBP = modify_const(const, cdataBP, True)\n",
+ "\n",
+ " if cdataBP.shape != cdata.shape:\n",
+ " print('Wrong bad pixel shape! {}, expected {}'.format(cdataBP.shape, cdata.shape))\n",
+ " continue\n",
+ "\n",
+ " # Apply bad pixel mask\n",
+ " cdataABP = np.copy(cdata)\n",
+ " cdataABP[cdataBP > 0] = np.nan\n",
+ "\n",
+ " # Create superpixels for constants with BP applied\n",
+ " cdataABP = get_rebined(cdataABP, spShape)\n",
+ " toStoreBP = prepare_to_store(np.nanmean(cdataABP, axis=(1, 3)), nMem)\n",
+ " toStoreBPStd = prepare_to_store(np.nanstd(cdataABP, axis=(1, 3)), nMem)\n",
+ "\n",
+ " # Prepare number of bad pixels per superpixels\n",
+ " cdataBP = get_rebined(cdataBP, spShape)\n",
+ " cdataNBP = prepare_to_store(np.nansum(cdataBP > 0, axis=(1, 3)), nMem)\n",
+ "\n",
+ " # Create superpixels for constants without BP applied\n",
+ " cdata = get_rebined(cdata, spShape)\n",
+ " toStoreStd = prepare_to_store(np.nanstd(cdata, axis=(1, 3)), nMem)\n",
+ " toStore = prepare_to_store(np.nanmean(cdata, axis=(1, 3)), nMem)\n",
+ "\n",
+ " print(\"Store values in dict\", const, qm, ctime)\n",
+ " ret_constants[const][qm].append({'ctime': ctime,\n",
+ " 'nBP': cdataNBP,\n",
+ " 'dataBP': toStoreBP,\n",
+ " 'dataBPStd': toStoreBPStd,\n",
+ " 'data': toStore,\n",
+ " 'dataStd': toStoreStd,\n",
+ " 'mdata': constant_parameters[i]}) \n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
"scrolled": true
},
"outputs": [],
@@ -333,7 +404,6 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
"scrolled": true
},
"outputs": [],
@@ -350,9 +420,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Combine FF and PC data to calculate Gain\n",
@@ -450,9 +518,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Parameters for plotting\n",
@@ -484,7 +550,6 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
"scrolled": true
},
"outputs": [],
@@ -524,9 +589,9 @@
" cmdata = np.array(data[\"mdata\"])\n",
" for i, tick in enumerate(ctimes_ticks):\n",
" ctimes_ticks[i] = ctimes_ticks[i] + \\\n",
- " ', V={:1.0f}'.format(cmdata[i]['Sensor Bias Voltage']) + \\\n",
+ " ', V={:1.0f}'.format(cmdata[i]['bias_voltage']) + \\\n",
" ', M={:1.0f}'.format(\n",
- " cmdata[i]['Memory cells'])\n",
+ " cmdata[i]['mem_cells'])\n",
"\n",
" sort_ind = np.argsort(ctimes_ticks)\n",
" ctimes_ticks = list(np.array(ctimes_ticks)[sort_ind])\n",
@@ -621,6 +686,13 @@
" out_folder, const, mod, gain, key),\n",
" vmin=vmin, vmax=vmax)"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {
@@ -639,7 +711,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.4.3"
+ "version": "3.6.7"
}
},
"nbformat": 4,