diff --git a/notebooks/FastCCD/CorrectionNotebook_NewDAQ_FastCCD_NBC.ipynb b/notebooks/FastCCD/CorrectionNotebook_NewDAQ_FastCCD_NBC.ipynb index c80d17a473346ec052092086a50245488adcca1a..5150634812d8d80c76d0c7e2770c2b81e3184911 100644 --- a/notebooks/FastCCD/CorrectionNotebook_NewDAQ_FastCCD_NBC.ipynb +++ b/notebooks/FastCCD/CorrectionNotebook_NewDAQ_FastCCD_NBC.ipynb @@ -18,16 +18,15 @@ "ExecuteTime": { "end_time": "2018-12-06T15:54:23.218849Z", "start_time": "2018-12-06T15:54:23.166497Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ - "in_folder = \"/gpfs/exfel/exp/SCS/201802/p002170/raw/\" # input folder, required\n", - "out_folder = '/gpfs/exfel/data/scratch/xcal/test/' # output folder, required\n", + "in_folder = \"/gpfs/exfel/exp/SCS/201930/p900074/raw/\" # input folder, required\n", + "out_folder = '/gpfs/exfel/data/scratch/ahmedk/test/fastccd/' # output folder, required\n", "path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # path template in hdf5 file\n", "path_inset = 'DA05'\n", - "run = 277 # run number\n", + "run = 448 # run number\n", "h5path = '/INSTRUMENT/SCS_CDIDET_FCCD2M/DAQ/FCCD:daqOutput/data/image' # path in HDF5 file\n", "h5path_t = '/CONTROL/SCS_CDIDET_FCCD2M/CTRL/LSLAN/inputA/crdg/value' # temperature path in HDF5 file\n", "h5path_cntrl = '/RUN/SCS_CDIDET_FCCD2M/DET/FCCD' # path to control data\n", @@ -52,7 +51,7 @@ "fix_temperature = 0. # fix temperature to this value, set to 0 to use slow control value\n", "flipped_between = [\"2019-02-01\", \"2019-04-02\"] # detector was flipped during this timespan\n", "temp_limits = 5 # limits within which temperature is considered the same\n", - "\n", + "commonModeAxis = 1 # Axis along which common mode will be calculated (0: along rows, 1: along columns)\n", "def balance_sequences(in_folder, run, sequences, sequences_per_node):\n", " import glob\n", " import re\n", @@ -187,17 +186,15 @@ "\n", "sensorSize = [x, y]\n", "chunkSize = 100 #Number of images to read per chunk\n", - "blockSize = [sensorSize[0]//2, sensorSize[1]//4] #Sensor area will be analysed according to blocksize\n", + "blockSize = [sensorSize[0]//2, sensorSize[1]] #Sensor area will be analysed according to blocksize\n", "xcal.defaultBlockSize = blockSize\n", "memoryCells = 1 #FastCCD has 1 memory cell\n", "#Specifies total number of images to proceed\n", "\n", "commonModeBlockSize = blockSize\n", - "commonModeAxisR = 'row'#Axis along which common mode will be calculated\n", "run_parallel = True\n", "profile = False\n", "\n", - "temperature_k = 291\n", "filename = fp_path.format(sequences[0] if sequences else 0)\n", "with h5py.File(filename, 'r') as f:\n", " bias_voltage = int(f['{}/biasclock/bias/value'.format(h5path_cntrl)][0])\n", @@ -227,8 +224,7 @@ "ExecuteTime": { "end_time": "2018-12-06T15:54:24.088948Z", "start_time": "2018-12-06T15:54:24.059925Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ @@ -393,7 +389,6 @@ " noiseMap = np.zeros(list(noise.data.shape)+[3], np.float32)\n", " noiseMap[...,i] = noise.data\n", "\n", - "\n", " ## bad pixels \n", "\n", " metadata = ConstantMetaData()\n", @@ -448,8 +443,7 @@ "ExecuteTime": { "end_time": "2018-12-06T15:54:28.343869Z", "start_time": "2018-12-06T15:54:28.271344Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ @@ -481,9 +475,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "relGainCA = copy.copy(relGain)\n", @@ -538,14 +530,12 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "import dateutil.parser\n", "flipped_between = [dateutil.parser.parse(d) for d in flipped_between]\n", - "flip_rgain = creation_time >= flipped_between[0] and creation_time <= flipped_between[1]\n", + "flip_rgain = creation_time.replace(tzinfo=None) >= flipped_between[0] and creation_time.replace(tzinfo=None) <= flipped_between[1]\n", "flip_rgain &= (metadata.calibration_constant_version.begin_at.replace(tzinfo=None) >= flipped_between[0] \n", " and metadata.calibration_constant_version.begin_at.replace(tzinfo=None) <= flipped_between[1])\n", "print(\"Accounting for flipped detector: {}\".format(flip_rgain))\n" @@ -558,21 +548,18 @@ "ExecuteTime": { "end_time": "2018-12-06T15:54:28.771629Z", "start_time": "2018-12-06T15:54:28.346051Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ "#************************Calculators************************#\n", - "\n", - "\n", - "cmCorrection = xcal.CommonModeCorrection([x, y], \n", + "cmCorrection = xcal.CommonModeCorrection([x, y],\n", " commonModeBlockSize, \n", - " commonModeAxisR,\n", - " nCells = memoryCells, \n", - " noiseMap = noiseMap,\n", - " runParallel=True,\n", - " stats=True)\n", + " commonModeAxis,\n", + " nCells = memoryCells,\n", + " stride=10,\n", + " runParallel=True,\n", + " stats=True, minFrac=0)\n", "\n", "patternClassifierLH = xcal.PatternClassifier([x//2, y], \n", " noiseMap[:x//2, :], \n", @@ -612,8 +599,7 @@ "ExecuteTime": { "end_time": "2018-12-06T16:08:51.886343Z", "start_time": "2018-12-06T16:08:51.842837Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ @@ -638,7 +624,14 @@ " range=[-50, 1000],\n", " nCells=memoryCells, \n", " cores=cpuCores,\n", - " blockSize=blockSize)" + " blockSize=blockSize)\n", + "\n", + "histCalCommonModeCor = xcal.HistogramCalculator([x, y], \n", + " bins=500, \n", + " range=[-50, 1000],\n", + " nCells=memoryCells, \n", + " cores=cpuCores,\n", + " blockSize=blockSize)" ] }, { @@ -655,8 +648,7 @@ "ExecuteTime": { "end_time": "2018-12-06T16:08:52.441784Z", "start_time": "2018-12-06T16:08:52.437284Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ @@ -673,12 +665,12 @@ "ExecuteTime": { "end_time": "2018-12-06T16:08:53.042555Z", "start_time": "2018-12-06T16:08:53.034522Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ "histCalOffsetCor.debug()\n", + "histCalCommonModeCor.debug()\n", "histCalPcorr.debug()\n" ] }, @@ -689,8 +681,7 @@ "ExecuteTime": { "end_time": "2018-12-06T16:08:53.551111Z", "start_time": "2018-12-06T16:08:53.531064Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ @@ -721,8 +712,7 @@ "ExecuteTime": { "end_time": "2018-12-06T16:10:55.917179Z", "start_time": "2018-12-06T16:09:01.603633Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ @@ -770,10 +760,10 @@ " chunks=(chunk_size_idim, oshape[1], oshape[2]),\n", " dtype=np.uint8, compression=\"gzip\")\n", " \n", + " # getting the 14th and 15th bit from the data, \n", + " # which containt gain before removing them from the data.\n", " gain = np.right_shift(data, 14)\n", - " \n", " gain[gain != 0] -= 1\n", - " \n", " fstride = 1\n", " if not flip_rgain: # rgain was taken during flipped orientation\n", " fstride = -1\n", @@ -788,7 +778,9 @@ " noise = np.choose(gain, (nmap[...,0], nmap[...,1], nmap[...,2]))\n", " bpix = np.choose(gain, (bmap[...,0], bmap[...,1], bmap[...,2]))\n", " \n", + " # Applying offset correction\n", " data -= offset\n", + "\n", " data *= rg\n", " \n", " if correct_offset_drift:\n", @@ -831,9 +823,14 @@ "\n", " patternClassifierLH._noisemap = noise[:x//2, :, :]\n", " patternClassifierUH._noisemap = noise[x//2:, :, :]\n", - "\n", - " data = cmCorrection.correct(data) # correct for the row common mode\n", + " \n", + " cellTable=np.zeros(data.shape[2], np.int32) # Common mode correction\n", + " data = cmCorrection.correct(data.astype(np.float32), cellTable=cellTable, noiseMap=noise) \n", + " \n", + " # correct for the row common mode\n", " ddsetcm[...] = np.moveaxis(data, 2, 0)\n", + " \n", + " histCalCommonModeCor.fill(data)\n", "\n", " dataLH = data[:x//2, :, :]\n", " dataUH = data[x//2:, :, :]\n", @@ -871,8 +868,7 @@ "ExecuteTime": { "end_time": "2018-12-06T16:10:56.094985Z", "start_time": "2018-12-06T16:10:55.918900Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ @@ -894,8 +890,7 @@ "ExecuteTime": { "end_time": "2018-12-06T16:10:56.126409Z", "start_time": "2018-12-06T16:10:56.096242Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ @@ -951,8 +946,7 @@ "ExecuteTime": { "end_time": "2018-12-06T16:10:56.176160Z", "start_time": "2018-12-06T16:10:56.127853Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ @@ -1008,8 +1002,7 @@ "ExecuteTime": { "end_time": "2018-12-06T16:10:56.190150Z", "start_time": "2018-12-06T16:10:56.177570Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ @@ -1041,8 +1034,7 @@ "ExecuteTime": { "end_time": "2018-12-06T16:10:56.203219Z", "start_time": "2018-12-06T16:10:56.191509Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ @@ -1070,8 +1062,7 @@ "ExecuteTime": { "end_time": "2018-12-06T16:10:56.212586Z", "start_time": "2018-12-06T16:10:56.204731Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ @@ -1097,14 +1088,12 @@ "ExecuteTime": { "end_time": "2018-12-06T16:13:12.889583Z", "start_time": "2018-12-06T16:13:11.122653Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ "ho,eo,co,so = histCalOffsetCor.get()\n", "\n", - "\n", "d = [{'x': co,\n", " 'y': ho,\n", " 'y_err': np.sqrt(ho[:]),\n", @@ -1113,10 +1102,19 @@ " 'errorcoarsing': 2,\n", " 'label': 'Offset corr.'\n", " },\n", - " \n", " ]\n", + " \n", + "if do_pattern_classification:\n", + " hcm,ecm,ccm,scm = histCalCommonModeCor.get()\n", + " d.append({'x': ccm,\n", + " 'y': hcm,\n", + " 'y_err': np.sqrt(hcm[:]),\n", + " 'drawstyle': 'steps-mid',\n", + " 'errorstyle': 'bars',\n", + " 'errorcoarsing': 2,\n", + " 'label': 'CommonMode corr.'\n", + " })\n", " \n", - "\n", "fig = xana.simplePlot(d, aspect=1, x_label='Energy(ADU)', \n", " y_label='Number of occurrences', figsize='2col',\n", " y_log=True, x_range=(-50,500),\n", @@ -1130,8 +1128,7 @@ "ExecuteTime": { "end_time": "2018-12-06T16:12:57.289742Z", "start_time": "2018-12-06T16:12:45.529734Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ @@ -1174,8 +1171,7 @@ "ExecuteTime": { "end_time": "2018-12-06T16:11:08.317130Z", "start_time": "2018-12-06T16:11:05.788655Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ @@ -1209,8 +1205,7 @@ "ExecuteTime": { "end_time": "2018-12-06T16:11:10.908912Z", "start_time": "2018-12-06T16:11:08.318486Z" - }, - "collapsed": true + } }, "outputs": [], "source": [ @@ -1231,9 +1226,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [] }