Clean outdated files

AGIPD/Characterize_AGIPD_Gain_Darks.py

-import sys
-# coding: utf-8
-
-# # Characterize Dark Images #
-# 
-# The following code analyzes a set of dark images taken with the AGIPD detector to deduce detector offsets and noise. Data for the detector's three gain stages needs to be present, separated into separate runs.
-# 
-# The notebook explicitely does what pyDetLib provides in its offset calculation method for streaming data.
-
-# In[23]:
-
-
-# imports and things that do not usually need to be changed
-from collections import OrderedDict
-import os
-import h5py
-import numpy as np
-import matplotlib
-matplotlib.use("Agg")
-import matplotlib.pyplot as plt
-# get_ipython().magic('matplotlib inline')
-
-# make sure a cluster is running with ipcluster start --n=32, give it a while to start
-from ipyparallel import Client
-profile  = str(sys.argv[-1])
-view = Client(profile=profile)[:]
-view.use_dill()
-
-from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions
-
-gains = np.arange(3)
-offset_runs = OrderedDict()
-
-# no need to change this
-
-QUADRANTS = 4
-MODULES_PER_QUAD = 4
-DET_FILE_INSET = "AGIPD"
-IL_MODE = False # or True for interlaced data
-
-# adapt this to the run being investigated
-in_folder  = str(sys.argv[1])
-offset_runs["high"]  = str(sys.argv[2])
-offset_runs["med"]  = str(sys.argv[3])
-offset_runs["low"]  = str(sys.argv[4])
-out_folder  = str(sys.argv[5])
-sequences  = [int(s) for s in sys.argv[6].split(',')]
-
-max_cells  = int(sys.argv[7])
-mem_cells = max_cells
-cells = np.arange(max_cells)
-
-local_output  = bool(sys.argv[8])
-db_output  = bool(sys.argv[9])
-bias_voltage  = int(sys.argv[10])
-cal_db_interface  = str(sys.argv[11])
-
-thresholds_offset_sigma = 3.
-thresholds_offset_hard = [4000, 8500]
-
-thresholds_noise_sigma = 5.
-thresholds_noise_hard = [4, 20]
-
-
-
-if not IL_MODE:
-    max_cells*=2
-
-
-# In[2]:
-
-
-def combine_stack(d, sdim):
-    """ A function for allowing to preview an assembled AGIPD image
-    """
-    combined = np.zeros((2048,2048, sdim))
-    combined[...] = np.nan
-    dy = 0
-    for i in range(16):
-        
-        if i < 8:
-            dx = -512
-            mx = 1
-            my = i % 8
-            combined[my*128+dy:(my+1)*128+dy,
-                     mx*512-dx:(mx+1)*512-dx, :] = d[i][:,::-1,:]
-            dy += 30
-            if i == 3:
-                dy += 30
-        elif i < 12:
-            dx = 100
-            if i == 8:
-                dy = 4*30 + 30 +50
-            
-            mx = 1
-            my = i % 8 +4
-            combined[my*128+dy:(my+1)*128+dy,
-                     mx*512-dx:(mx+1)*512-dx, :] = d[i][::-1,:,:]
-            dy += 30
-        else:
-            dx = 100
-            if i == 11:
-                dy = 50
-            
-            mx = 1
-            my = i - 14
-            
-            combined[my*128+dy:(my+1)*128+dy,
-                     mx*512-dx:(mx+1)*512-dx, :] = d[i][::-1,:,:]
-            dy += 30
-        
-    return combined
-
-
-# The following lines will create a queue of files which will the be executed module-parallel. Distiguishing between different gains.
-
-# In[3]:
-
-
-# set everything up filewise
-from queue import Queue
-if not os.path.exists(out_folder):
-    os.makedirs(out_folder)
-    
-def map_modules_from_files(filelist):
-    module_files = {}
-    mod_ids = {}
-    for quadrant in range(0, QUADRANTS):
-        for module in range(0, MODULES_PER_QUAD):
-            name = "Q{}M{}".format(quadrant + 1, module + 1)
-            module_files[name] = Queue()
-            num = quadrant * 4 + module
-            mod_ids[name] = num
-            file_infix = "{}{:02d}".format(DET_FILE_INSET, num)
-            for file in filelist:
-                if file_infix in file:
-                    module_files[name].put(file)
-    return module_files, mod_ids
-
-gain_mapped_files = OrderedDict()
-for gain, run in offset_runs.items():
-    ginfolder = "{}/{}".format(in_folder, run)
-    dirlist = os.listdir(ginfolder)
-    file_list = []
-    for entry in dirlist:
-        #only h5 file
-        abs_entry = "{}/{}".format(ginfolder, entry)
-        if os.path.isfile(abs_entry) and os.path.splitext(abs_entry)[1] == ".h5":
-
-            if sequences is None:
-                file_list.append(abs_entry)
-            else:
-                for seq in sequences:
-                    if "{:05d}.h5".format(seq) in abs_entry:
-                        file_list.append(os.path.abspath(abs_entry))
-
-    mapped_files, mod_ids = map_modules_from_files(file_list)
-    gain_mapped_files[gain] = mapped_files
-
-
-# ## Calculate Offsets, Noise and Thresholds ##
-# 
-# The calculation is performed per-pixel and per-memory-cell. Offsets are simply the median value for a set of dark data taken at a given gain, noise the standard deviation, and gain-bit values the medians of the gain array.
-
-# In[4]:
-
-
-import copy
-from functools import partial
-def characterize_module(il_mode, cells, bp_thresh, inp):
-    import numpy as np
-    import copy
-    import h5py
-    
-    filename, filename_out, channel = inp
-    thresholds_offset_hard, thresholds_offset_sigma, thresholds_noise_hard, thresholds_noise_sigma = bp_thresh 
-
-    infile = h5py.File(filename, "r", driver="core")
-    count = np.squeeze(infile["/INDEX/SPB_DET_AGIPD1M-1/DET/{}CH0:xtdf/image/count".format(channel)])
-    first = np.squeeze(infile["/INDEX/SPB_DET_AGIPD1M-1/DET/{}CH0:xtdf/image/first".format(channel)])
-    last_index = int(first[count != 0][-1]+count[count != 0][-1])
-    first_index = int(first[count != 0][0])
-    im = np.array(infile["/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/{}CH0:xtdf/image/data".format(channel)][first_index:last_index,...])    
-    infile.close()
-
-    if il_mode:
-        ga = im[1::2, 0, ...]
-        im = im[0::2, 0, ...].astype(np.float32)
-    else:
-        ga = im[:, 1, ...]
-        im = im[:, 0, ...].astype(np.float32)
-
-    im = np.rollaxis(im, 2)
-    im = np.rollaxis(im, 2, 1)
-
-    ga = np.rollaxis(ga, 2)
-    ga = np.rollaxis(ga, 2, 1)
-
-    offset = np.zeros((im.shape[0], im.shape[1], cells//2))
-    gains = np.zeros((im.shape[0], im.shape[1], cells//2))
-    noise = np.zeros((im.shape[0], im.shape[1], cells//2))
-    
-    for cc in range(cells//2):
-        
-        offset[...,cc] = np.median(im[..., cc::cells//2], axis=2)
-        noise[...,cc] = np.std(im[..., cc::cells//2], axis=2)
-        gains[...,cc] = np.median(ga[..., cc::cells//2], axis=2)
-        
-    # bad pixels
-    bp = np.zeros(offset.shape, np.uint8)
-    # offset related bad pixels
-    offset_mn = np.nanmedian(offset, axis=(0,1))
-    offset_std = np.nanstd(offset, axis=(0,1))    
-    
-    bp[(offset < offset_mn-thresholds_offset_sigma*offset_std) |
-       (offset > offset_mn+thresholds_offset_sigma*offset_std)] |= 2**0
-    bp[(offset < thresholds_offset_hard[0]) | (offset > thresholds_offset_hard[1])] |= 2**0
-    bp[~np.isfinite(offset)] |= 2**0
-    
-    # noise related bad pixels
-    noise_mn = np.nanmedian(noise, axis=(0,1))
-    noise_std = np.nanstd(noise, axis=(0,1))    
-    
-    bp[(noise < noise_mn-thresholds_noise_sigma*noise_std) |
-       (noise > noise_mn+thresholds_noise_sigma*noise_std)] |= 2**4
-    bp[(noise < thresholds_noise_hard[0]) | (noise > thresholds_noise_hard[1])] |= 2**4
-    bp[~np.isfinite(noise)] |= 2**4
-        
-    return offset, noise, gains, bp
-        
-        
-offset_g = {}
-noise_g = {}
-gain_g = {}
-badpix_g = {}
-gg = 0
-for gain, mapped_files in gain_mapped_files.items():
-    
-    inp = []
-    dones = []
-    for i in range(16):
-        qm = "Q{}M{}".format(i//4 +1, i % 4 + 1)    
-        if qm in mapped_files and not mapped_files[qm].empty():
-            fname_in = mapped_files[qm].get()            
-            dones.append(mapped_files[qm].empty())
-        else:
-            continue
-        fout = os.path.abspath("{}/{}".format(out_folder, (os.path.split(fname_in)[-1]).replace("RAW", "CORR")))
-        inp.append((fname_in, fout, i))
-    first = False
-    p = partial(characterize_module, IL_MODE, max_cells,
-               (thresholds_offset_hard, thresholds_offset_sigma,
-                thresholds_noise_hard, thresholds_noise_sigma))
-    #results = list(map(p, inp))
-    results = view.map_sync(p, inp)
-    for i, r in enumerate(results):
-        offset, noise, gain, bp = r
-        qm = "Q{}M{}".format(i//4 +1, i % 4 + 1)
-        if qm not in offset_g:
-            offset_g[qm] = np.zeros((offset.shape[0], offset.shape[1], offset.shape[2], 3))
-            noise_g[qm] = np.zeros_like(offset_g[qm])
-            gain_g[qm] = np.zeros_like(offset_g[qm])
-            badpix_g[qm] = np.zeros_like(offset_g[qm])
-        
-        offset_g[qm][...,gg] = offset
-        noise_g[qm][...,gg] = noise
-        gain_g[qm][...,gg] = gain
-        badpix_g[qm][...,gg] = bp
-    gg +=1
-
-
-# The thresholds for gain switching are then defined as the mean value between in individual gain bit levels. Note that these thresholds need to be refined with charge induced thresholds, as the two are not the same.
-
-# In[5]:
-
-
-thresholds_g = {}
-for qm in gain_g.keys():
-    thresholds_g[qm] = np.zeros((gain_g[qm].shape[0], gain_g[qm].shape[1], gain_g[qm].shape[2], 2))
-    thresholds_g[qm][...,0] = (gain_g[qm][...,1]+gain_g[qm][...,0])/2
-    thresholds_g[qm][...,1] = (gain_g[qm][...,2]+gain_g[qm][...,1])/2
-
-
-# The following code is for inspection purposes, it will render the constants for each module and gain and memory cell.
-
-# In[6]:
-
-
-import matplotlib.pyplot as plt
-from mpl_toolkits.axes_grid1 import AxesGrid
-
-res = {}
-for i in range(16):
-    qm = "Q{}M{}".format(i//4+1, i%4+1)
-    res[qm] = {'Offset': offset_g[qm],
-               'Noise': noise_g[qm],
-               'Threshold': thresholds_g[qm],
-               'BadPixels': badpix_g[qm]    
-               }
-
-def show_overview(cell_to_preview, gain_to_preview):
-    
-    for module, data in res.items():
-        fig = plt.figure(figsize=(40,40))
-        grid = AxesGrid(fig, 111,
-                        nrows_ncols=(2, 2),
-                        axes_pad=(0.9, 0.15),
-                        label_mode="1",
-                        share_all=True,
-                        cbar_location="right",
-                        cbar_mode="each",
-                        cbar_size="7%",
-                        cbar_pad="2%",
-                        )
-        i = 0
-        for key, item in data.items():
-            cf = 0
-            if "Threshold" in key:
-                cf = -1
-            if len(item.shape) == 4:
-                med = np.nanmedian(item[...,cell_to_preview, gain_to_preview + cf])
-            else:
-                med = np.nanmedian(item[...,cell_to_preview])
-                
-            bound = 0.2
-            while(np.count_nonzero((item[...,cell_to_preview, gain_to_preview + cf] < med-np.abs(bound*med)) |
-                                   (item[...,cell_to_preview, gain_to_preview + cf] > med+np.abs(bound*med)))/item[...,cell_to_preview, gain_to_preview + cf].size > 0.01):            
-                bound *=2
-            
-            if "BadPixels" in key:
-                im = grid[i].imshow(np.log2(item[...,cell_to_preview, gain_to_preview + cf]), interpolation="nearest",
-                                    vmin=0, vmax=8, aspect='auto')
-            else:
-                if len(item.shape) == 4:
-                    im = grid[i].imshow(item[...,cell_to_preview, gain_to_preview + cf], interpolation="nearest",
-                                       vmin=med-np.abs(bound*med), vmax=np.abs(med+bound*med), aspect='auto')
-                else:
-                    im = grid[i].imshow(item[...,cell_to_preview], interpolation="nearest",
-                                       vmin=med-np.abs(bound*med), vmax=med+np.abs(bound*med), aspect='auto')
-            cb = grid.cbar_axes[i].colorbar(im)
-
-            grid[i].text(20, 50, key, color="w" if key != "BadPixels" else "k", fontsize=50)
-
-            i += 1
-        grid[0].text(5, 20, module, color="r" if key != "BadPixels" else "k", fontsize=20)
-        fig.savefig("{}/dark_analysis_{}_module_{}.png".format(out_folder,
-                                                               "_".join(offset_runs.values()),
-                                                               module))
-
-
-# In[7]:
-
-
-show_overview(12, 0)
-
-
-# In[8]:
-
-
-show_overview(12, 1)
-
-
-# In[9]:
-
-
-show_overview(4, 2)
-
-
-# In[10]:
-
-
-channel_mapping = {}
-for i in range(16):
-    qm = "Q{}M{}".format(i//4 +1, i % 4 + 1)
-    channel_mapping[qm] = i 
-    
-def create_constant_overview(constant, name, vmin, vmax, entries=3):
-    for g in range(entries):
-        fig = plt.figure(figsize=(10, 5))
-        ax = fig.add_subplot(111)
-        for qm in constant.keys():
-            d = constant[qm][...,g]
-            print("{} {}, gain {:0.2f}: mean: {:0.2f}, median: {:0.2f}, std: {:0.2f}".format(name, qm, g,
-                                                                      np.mean(d),
-                                                                      np.median(d),
-                                                                      np.std(d)))
-            
-            ax.step(np.arange(max_cells//2), np.median(d, axis=(0,1)))
-        fig.savefig("{}/dark_analysis_{}_{}_per_cell_gain{}.png".format(out_folder,
-                                                                        "_".join(offset_runs.values()),
-                                                                        name, g))
-
-
-# In[11]:
-
-
-create_constant_overview(offset_g, "Offset", 4000, 8000)
-
-
-# In[12]:
-
-
-create_constant_overview(noise_g, "Noise", 0, 100)
-
-
-# In[13]:
-
-
-create_constant_overview(thresholds_g, "Threshold", 3000, 8000, 2)
-
-
-# Finally, we persist the data.
-
-# In[14]:
-
-
-if local_output:
-    ofile = "{}/agipd_offset_store_{}.h5".format(out_folder, "_".join(offset_runs.values()))
-    store_file = h5py.File(ofile, "w")
-    for qm in offset_g.keys():
-        store_file["{}/Offset/0/data".format(qm)] = offset_g[qm]
-        store_file["{}/Noise/0/data".format(qm)] = noise_g[qm]
-        store_file["{}/Threshold/0/data".format(qm)] = thresholds_g[qm]
-        store_file["{}/BadPixels/0/data".format(qm)] = badpix_g[qm]
-    store_file.close()
-
-
-# In[30]:
-
-
-from time import sleep
-if db_output:
-    for qm in offset_g.keys():
-        metadata = ConstantMetaData()
-        offset = Constants.AGIPD.Offset()
-        offset.data = offset_g[qm]
-        metadata.calibration_constant = offset
-
-        # set the operating condition
-        condition = Conditions.Dark.AGIPD(memory_cells=mem_cells, bias_voltage=bias_voltage)
-        device = getattr(Detectors.AGIPD1M1, qm)
-        uuid = device.uuid
-        condition.name = "Default AGIPD Condition - Cells {} - UUID: {}".format(mem_cells, int(uuid))
-        metadata.detector_condition = condition
-        
-        # specify the a version for this constant
-        metadata.calibration_constant_version = Versions.Now(device=device)
-        
-        metadata.send(cal_db_interface)
-        
-        metadata = ConstantMetaData()
-        noise = Constants.AGIPD.Noise()
-        noise.data = noise_g[qm]
-        metadata.calibration_constant = noise
-
-        # set the operating condition
-        condition = Conditions.Dark.AGIPD(memory_cells=mem_cells, bias_voltage=bias_voltage)
-        condition.name = "Default AGIPD Condition - Cells {} - UUID: {}".format(mem_cells, int(uuid))
-        metadata.detector_condition = condition
-
-        # specify the a version for this constant
-        metadata.calibration_constant_version = Versions.Now(device=device)
-        metadata.send(cal_db_interface)
-        
-        metadata = ConstantMetaData()
-        thresholds = Constants.AGIPD.ThresholdsDark()
-        thresholds.data = thresholds_g[qm]
-        metadata.calibration_constant = thresholds
-
-        # set the operating condition
-        condition = Conditions.Dark.AGIPD(memory_cells=mem_cells, bias_voltage=bias_voltage)
-        condition.name = "Default AGIPD Condition - Cells {} - UUID: {}".format(mem_cells, int(uuid))
-        metadata.detector_condition = condition
-
-        # specify the a version for this constant
-        metadata.calibration_constant_version = Versions.Now(device=device)
-        metadata.send(cal_db_interface)
-        
-
-
-# In[ ]:
-
-
-
-

AGIPD/finalize.sh

-#!/bin/bash
-echo 'Running finalize script'
-python3 -c "from slurm_tools import finalize; finalize(['974134', '974135', '974136', '974137', '974138', '974139', '974140', '974141', '974142', '974143', '974144', '974145', '974146', '974147', '974148', '974149'], '/gpfs/exfel/data/user/haufs/scripted_offline_cal/pycalibration/AGIPD/slurm_tmp_e3f3a6c7-8aea-44fd-956f-572ae71b4812', 'AGIPD Calibration', 'AGIPD Flat Fields', 'S. Hauf', '0.1')"

AGIPD/parallel_hist.py

-in_folder = "/gpfs/exfel/exp/SPB/201701/"
-proposals = ["p002013"]
-detector = "AGIPD"
-
-import warnings
-warnings.filterwarnings('ignore')
-
-import copy
-import os
-import h5py
-import numpy as np
-import sys
-from functools import partial
-
-files = sys.argv[2].split(",")
-profile = sys.argv[3]
-outpath = sys.argv[4]
-modules = sys.argv[5].split(",")
-
-
-# parallel processing via ipcluster
-# make sure a cluster is running with ipcluster start --n=32, give it a while to start
-from ipyparallel import Client
-client = Client(profile=profile)
-view = client[:]
-view.use_dill()
-
-
-
-mhists = np.zeros((74, 64, 500, 500))
-    
-
-for i, file in enumerate(files):
-    module = int(modules[i])
-    print("Processing {} to {} on {} for module {}".format(file, outpath, profile, module))
-    try:
-        
-        infile = h5py.File(file, "r")
-        cells = np.squeeze(infile["/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/{}CH0:xtdf/image/cellId".format(module)][()])
-        infile.close()
-        
-        def do_cellhist(file, module, cells, cell):
-            import numpy as np
-            import h5py
-            cidx = cells == cell
-            infile = h5py.File(file, "r")
-            icell = infile["/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/{}CH0:xtdf/image/data".format(module)][cidx,0,:,:64]
-            gcell = infile["/INSTRUMENT/SPB_DET_AGIPD1M-1/DET/{}CH0:xtdf/image/data".format(module)][cidx,1,:,:64]
-            infile.close()
-            rowh = []
-            for row in range(64):
-
-                h, _, _ = np.histogram2d(gcell[...,row].flatten(),
-                                         icell[...,row].flatten(), bins=(500, 500),
-                                         range=((0,12000),(0,12000)))
-                rowh.append(h)
-            return rowh
-            
-            
-        mapped_data = [c for c in np.unique(cells)]
-        res = view.map_sync(partial(do_cellhist, file, module, cells), mapped_data)
-            
-        for cell, cellh in enumerate(res):
-            for row, h in enumerate(cellh):
-                mhists[cell, row, ...] += h                
-                        
-            
-    except Exception as e:
-        print(e)
-        pass
-    
-        
-np.savez_compressed("{}/mod{}_{}".format(outpath, module, profile), mhists)
\ No newline at end of file

AGIPD/run_parallel_hist.py

-import sys
-in_folder = sys.argv[1] #"/gpfs/exfel/exp/SPB/201701/"
-proposals = sys.argv[2].split(",") #["p002013"]
-out_path = sys.argv[3]
-exclude = sys.argv[4]
-detector = "AGIPD"
-files_per_node = 50
-
-import warnings
-warnings.filterwarnings('ignore')
-
-import copy
-import os
-import os
-from subprocess import Popen, PIPE
-from time import sleep
-import time
-from uuid import uuid4
-
-modules = range(3,4)
-
-excludes = []
-for ex in exclude.split(","):
-    if "-" in ex:
-        start, end = ex.split("-")
-        
-        excludes += ["r{:04d}".format(r) for r in range(int(start), int(end))]
-    else:
-        excludes.append("r{:04d}".format(int(ex)))
-        
-print("Excluding: {}".format(", ".join(excludes)))
-
-if proposals == "all":
-    proposals = os.listdir(in_folder)
-
-files = []
-for proposal in proposals:
-    pfolder = "{}/{}/raw".format(in_folder, proposal)
-    runs = os.listdir(pfolder)
-    for run in runs:
-        if run in excludes:
-            continue
-        rfolder = "{}/{}".format(pfolder, run)
-        for file in os.listdir(rfolder):
-            for module in modules:
-                fcheck = "RAW-{}-{}{:02d}".format(run.upper(), detector, module)
-                if fcheck in file:
-                    files.append((module, "{}/{}".format(rfolder, file)))
-
-print("Working on {} files".format(len(files)))
-
-srun_base = ["sbatch", "-p", "exfel", "-t", "04:00:00"]
-
-srun_base += [os.path.abspath("{}/run_parallel_hist.sh".format(os.getcwd())),
-              os.path.abspath("{}/parallel_hist.py".format(os.getcwd())),
-              in_folder
-             ]
-for i in range(len(files)//files_per_node):
-    fls = []
-    modules = []
-    for fm in files[i*files_per_node:(i+1)*files_per_node]:
-        module, file = fm
-        fls.append(file)
-        modules.append(str(module))
-    s_run = copy.copy(srun_base)
-    s_run += [",".join(fls), "parallel_hist_{}".format(uuid4()), out_path, ",".join(modules), exclude]
-    Popen(s_run)
\ No newline at end of file

AGIPD/run_parallel_hist.sh

-#!/bin/bash
-#SBATCH -N 1 # number of nodes
-#SBATCH -n 32 # number of cores
-#module load python3
-#~/.local/bin/ipcluster start --n=32 --daemon --ip="*"
-source /gpfs/exfel/data/user/haufs/karabo/activate
-ipython profile create ${4} --parallel
-/gpfs/exfel/data/user/haufs/karabo/extern/bin/ipcluster start --n=32 --profile=${4} --daemon &
-sleep 30
-/gpfs/exfel/data/user/haufs/karabo/extern/bin/python "$@"
-/gpfs/exfel/data/user/haufs/karabo/extern/bin/ipcluster stop --profile=${4}
-rm -rf "/home/haufs/.ipython/profile_${4}"
-#echo ${11}
\ No newline at end of file

AGIPD/slurm_CI.py

-import argparse
-import copy
-import glob
-import os
-from subprocess import Popen, PIPE
-from time import sleep
-import time
-from uuid import uuid4
-
-parser = argparse.ArgumentParser(description="Main entry point "
-                                             "for offline calibration")
-parser.add_argument("--rawpath", type=str)
-parser.add_argument("--output", type=str)
-parser.add_argument("--mem-cells", type=str)
-parser.add_argument("--runs", type=str)
-parser.add_argument("--sequences", type=str)
-parser.add_argument("--interlaced", action="store_true", default=False)
-parser.add_argument("--modules", type=str, default="all")
-parser.add_argument("--nodb", action="store_true", default=False)
-parser.add_argument("--bias", type=str, default=500)
-parser.add_argument("--dbhost", type=str, default="tcp://max-exfl015:5005")
-
-
-def notebook_to_python():
-    nb_name = "Chracterize_AGIPD_Gain_PC.ipynb"
-    conv = ["jupyter", "nbconvert", "--to", "python",
-            nb_name, "--output", "conv_tmp"]
-    
-    Popen(conv).wait()
-    
-    mapping = {}
-    arg_cnt = 1
-    has_profile = False
-    with open("./conv_tmp.py", "r") as infile:
-        with open(nb_name.replace("ipynb", "py"), "w") as outfile:
-            outfile.write("import sys")
-            for line in infile.readlines():
-                if "SLURMHINT" in line:
-                    
-                    line, hint = line.split("#")
-                    field, assign = line.split("=")
-                    parm = hint.split("SLURMHINT:")[1]
-                    parm, typ = parm.split(",")
-                    parm = parm.strip()
-                    if parm == "profile":
-                        mapping[-1] = parm
-                        line = "{field} = {typ}(sys.argv[{arg_cnt}])\n".format(field=field, 
-                                                                                 typ=typ.strip(),
-                                                                                 arg_cnt=-1)
-                    else:
-                        mapping[arg_cnt] = parm
-                        if typ.strip() != "list":
-                            line = "{field} = {typ}(sys.argv[{arg_cnt}])\n".format(field=field, 
-                                                                                 typ=typ.strip(),
-                                                                                 arg_cnt=arg_cnt)
-                        else:
-                            line = "{field} = [int(s) for s in sys.argv[{arg_cnt}].split(',')]\n".format(field=field, arg_cnt=arg_cnt)
-                        arg_cnt += 1
-                    outfile.write(line)
-                else:
-                    if "get_ipython()" in line:
-                        line = "# "+line
-                    outfile.write(line)
-    
-    return mapping
-    
-
-def run():
-    args = vars(parser.parse_args())
-    path_temp = "{}/r{{:04d}}".format(args["rawpath"])
-    out_folder= args["output"]
-    cells = args["mem_cells"]
-    il_mode = bool(args["interlaced"])
-    mods = args["modules"]
-    runs_in = args["runs"]
-    sequences = args["sequences"]
-    modules = range(16) if mods.upper()=="ALL" else [int(m) for m in mods.split(",")]
-    nodb = bool(args["nodb"])
-    bias = args["bias"]
-    dbhost = args["dbhost"]
-  
-    runs = []
-    for rcomp in runs_in.split(","):
-        if "-" in rcomp:
-            start, end = rcomp.split("-")
-            runs += list(range(int(start), int(end)))
-        else:
-            runs += [int(rcomp)]
-    
-    
-    out_mapping = notebook_to_python()
-        
-    for module in modules:
-        in_mapping = {"maxcells": cells,
-                      "cells": cells,
-                      "path_temp": path_temp,
-                      "il_mode": il_mode,
-                      "modules": module,
-                      "runs": ",".join([str(r) for r in runs]),
-                      "out_folder": out_folder,
-                      "seq": sequences,
-                      "local_output": True if nodb else False,
-                      "db_output": False if nodb else True,
-                      "bias_voltage": bias,
-                      "db_host": dbhost
-                      }
-        
-        srun_base = ["sbatch", "-p", "exfel", "-t", "24:00:00"]
-
-        srun_base += [os.path.abspath("{}/slurm_CI.sh".format(os.getcwd())),
-                      os.path.abspath("{}/Chracterize_AGIPD_Gain_PC.py".format(os.getcwd()))]
-
-        for key in sorted(out_mapping.keys()):
-            if out_mapping[key] in in_mapping:
-                srun_base.append(str(in_mapping[out_mapping[key]]))
-        Popen(srun_base).wait()
-
-if __name__ == "__main__":
-    run()
-
-
-

AGIPD/slurm_CI.sh

-#!/bin/bash
-source /gpfs/exfel/data/user/haufs/karabo/activate
-uuid=$(uuidgen)
-echo "File is: ${2}"
-echo ${uuid} >> $2
-export MPLBACKEND=AGG
-ipython profile create ${uuid} --parallel
-/gpfs/exfel/data/user/haufs/karabo/extern/bin/ipcluster start --n=32 --profile=${uuid} --daemon &
-sleep 30
-echo "Running script"
-jupyter nbconvert --to rst --ExecutePreprocessor.timeout=36000 --ExecutePreprocessor.allow_errors=True --TemplateExporter.exclude_input=True --execute $1
-/gpfs/exfel/data/user/haufs/karabo/extern/bin/ipcluster stop --profile=${uuid}
-rm -rf "/home/haufs/.ipython/profile_${uuid}"
-