Load.py

# -*- coding: utf-8 -*-
""" Toolbox for SCS.

    Various utilities function to quickly process data measured at the SCS instruments.

    Copyright (2019) SCS Team.
"""
import numpy as np
from karabo_data import by_index, RunDirectory
from karabo_data.read_machinery import find_proposal
import xarray as xr
import os

mnemonics = {
    # Machine
    "sase3": {'source':'SCS_RR_UTC/MDL/BUNCH_DECODER',
              'key':'sase3.pulseIds.value',
              'dim':['bunchId']},
    "sase2": {'source':'SCS_RR_UTC/MDL/BUNCH_DECODER',
              'key':'sase2.pulseIds.value',
              'dim':['bunchId']},
    "sase1": {'source':'SCS_RR_UTC/MDL/BUNCH_DECODER',
              'key':'sase1.pulseIds.value',
              'dim':['bunchId']},
    "maindump": {'source':'SCS_RR_UTC/MDL/BUNCH_DECODER',
                 'key':'maindump.pulseIds.value',
                 'dim':['bunchId']},
    "bunchpattern": {'source':'SCS_RR_UTC/TSYS/TIMESERVER',
                     'key':'readBunchPatternTable.value',
                     'dim':None},
    "bunchPatternTable": {'source':'SCS_RR_UTC/TSYS/TIMESERVER',
                     'key':'bunchPatternTable.value',
                     'dim':['pulse_slot']},
    "npulses_sase3": {'source':'SCS_RR_UTC/MDL/BUNCH_DECODER',
                      'key':'sase3.nPulses.value',
                      'dim':None},
    "npulses_sase1": {'source':'SCS_RR_UTC/MDL/BUNCH_DECODER',
                      'key':'sase1.nPulses.value',
                      'dim':None},
    # SA3
    "nrj": {'source':'SA3_XTD10_MONO/MDL/PHOTON_ENERGY',
            'key':'actualEnergy.value',
            'dim':None},

    "M2BEND": {'source': 'SA3_XTD10_MIRR-2/MOTOR/BENDER',
               'key': 'actualPosition.value',
               'dim':None},
    "VSLIT": {'source':'SA3_XTD10_VSLIT/MDL/BLADE',
              'key':'actualGap.value',
              'dim':None},
    "ESLIT": {'source':'SCS_XTD10_ESLIT/MDL/MAIN',
              'key':'actualGap.value',
              'dim':None},
    "HSLIT": {'source':'SCS_XTD10_HSLIT/MDL/BLADE',
              'key':'actualGap.value',
              'dim':None},
    "transmission": {'source':'SA3_XTD10_GATT/MDL/GATT_TRANSMISSION_MONITOR',
                     'key':'Estimated_Tr.value',
                     'dim':None},
    "GATT_pressure": {'source':'P_GATT',
                      'key':'value.value',
                      'dim':None},
    "navitar": {'source':'SCS_XTD10_IMGES/CAM/BEAMVIEW_NAVITAR:daqOutput',
               'key':'data.image.pixels',
               'dim':['x','y']},
    "UND": {'source':'SA3_XTD10_UND/DOOCS/PHOTON_ENERGY',
                'key':'actualPosition.value',
                'dim':None},
    # XTD10 XGM
    ## keithley
    "XTD10_photonFlux": {'source':'SA3_XTD10_XGM/XGM/DOOCS',
                     'key':'pulseEnergy.photonFlux.value',
                     'dim':None},
    "XTD10_photonFlux_sigma": {'source':'SA3_XTD10_XGM/XGM/DOOCS',
                     'key':'pulseEnergy.photonFluxSigma.value',
                     'dim':None},
    ## ADC
    "XTD10_XGM": {'source':'SA3_XTD10_XGM/XGM/DOOCS:output',
                'key':'data.intensityTD',
                'dim':['XGMbunchId']},
    "XTD10_XGM_sigma": {'source':'SA3_XTD10_XGM/XGM/DOOCS:output',
                'key':'data.intensitySigmaTD',
                'dim':['XGMbunchId']},
    "XTD10_SA3": {'source':'SA3_XTD10_XGM/XGM/DOOCS:output',
                'key':'data.intensitySa3TD',
                'dim':['XGMbunchId']},
    "XTD10_SA3_sigma": {'source':'SA3_XTD10_XGM/XGM/DOOCS:output',
                'key':'data.intensitySa3SigmaTD',
                'dim':['XGMbunchId']},
    "XTD10_SA1": {'source':'SA3_XTD10_XGM/XGM/DOOCS:output',
                'key':'data.intensitySa1TD',
                'dim':['XGMbunchId']},
    "XTD10_SA1_sigma": {'source':'SA3_XTD10_XGM/XGM/DOOCS:output',
                'key':'data.intensitySa1SigmaTD',
                'dim':['XGMbunchId']},
    ## low pass averaged ADC
    "XTD10_slowTrain": {'source':'SA3_XTD10_XGM/XGM/DOOCS',
                    'key':'controlData.slowTrain.value',
                    'dim':None},
    "XTD10_slowTrain_SA1": {'source':'SA3_XTD10_XGM/XGM/DOOCS',
                    'key':'controlData.slowTrainSa1.value',
                    'dim':None},
    "XTD10_slowTrain_SA3": {'source':'SA3_XTD10_XGM/XGM/DOOCS',
                    'key':'controlData.slowTrainSa3.value',
                    'dim':None},

    # SCS XGM
    ## keithley
    "SCS_photonFlux": {'source':'SCS_BLU_XGM/XGM/DOOCS',
                     'key':'pulseEnergy.photonFlux.value',
                     'dim':None},
    "SCS_photonFlux_sigma": {'source':'SCS_BLU_XGM/XGM/DOOCS',
                     'key':'pulseEnergy.photonFluxSigma.value',
                     'dim':None},
    ## ADC
    "SCS_XGM": {'source':'SCS_BLU_XGM/XGM/DOOCS:output',
                'key':'data.intensityTD',
                'dim':['XGMbunchId']},
    "SCS_XGM_sigma": {'source':'SCS_BLU_XGM/XGM/DOOCS:output',
                'key':'data.intensitySigmaTD',
                'dim':['XGMbunchId']},
    "SCS_SA1": {'source':'SCS_BLU_XGM/XGM/DOOCS:output',
                'key':'data.intensitySa1TD',
                'dim':['XGMbunchId']},
    "SCS_SA1_sigma": {'source':'SCS_BLU_XGM/XGM/DOOCS:output',
                'key':'data.intensitySa1SigmaTD',
                'dim':['XGMbunchId']},
    "SCS_SA3": {'source':'SCS_BLU_XGM/XGM/DOOCS:output',
                'key':'data.intensitySa3TD',
                'dim':['XGMbunchId']},
    "SCS_SA3_sigma": {'source':'SCS_BLU_XGM/XGM/DOOCS:output',
                'key':'data.intensitySa3SigmaTD',
                'dim':['XGMbunchId']},
    ## low pass averaged ADC
    "SCS_slowTrain": {'source':'SCS_BLU_XGM/XGM/DOOCS',
                    'key':'controlData.slowTrain.value',
                    'dim':None},
    "SCS_slowTrain_SA1": {'source':'SCS_BLU_XGM/XGM/DOOCS',
                    'key':'controlData.slowTrainSa1.value',
                    'dim':None},
    "SCS_slowTrain_SA3": {'source':'SCS_BLU_XGM/XGM/DOOCS',
                    'key':'controlData.slowTrainSa3.value',
                    'dim':None},

    # KBS
    "HFM_CAPB": {'source':'SCS_KBS_HFM/ASENS/CAPB',
                 'key':'value.value',
                 'dim':None},
    "HFM_CAPF": {'source':'SCS_KBS_HFM/ASENS/CAPF',
                 'key':'value.value',
                 'dim':None},
    "HFM_CAPM": {'source':'SCS_KBS_HFM/ASENS/CAPM',
                 'key':'value.value',
                 'dim':None},
    "HFM_BENDERB": {'source':'SCS_KBS_HFM/MOTOR/BENDERB',
                    'key':'encoderPosition.value',
                    'dim':None},
    "HFM_BENDERF": {'source':'SCS_KBS_HFM/MOTOR/BENDERF',
                    'key':'encoderPosition.value',
                    'dim':None},
    "VFM_CAPB": {'source':'SCS_KBS_VFM/ASENS/CAPB',
                 'key':'value.value',
                 'dim':None},
    "VFM_CAPF": {'source':'SCS_KBS_VFM/ASENS/CAPF',
                 'key':'value.value',
                 'dim':None},
    "VFM_CAPM": {'source':'SCS_KBS_VFM/ASENS/CAPM',
                 'key':'value.value',
                 'dim':None},
    "VFM_BENDERB": {'source':'SCS_KBS_VFM/MOTOR/BENDERB',
                    'key':'encoderPosition.value',
                    'dim':None},
    "VFM_BENDERF": {'source':'SCS_KBS_VFM/MOTOR/BENDERF',
                    'key':'encoderPosition.value',
                    'dim':None},
    
    # AFS LASER
    "AFS_PhaseShifter": {'source':'SCS_ILH_LAS/PHASESHIFTER/DOOCS',
                 'key':'actualPosition.value',
                 'dim':None},
    "AFS_DelayLine": {'source':'SCS_ILH_LAS/MOTOR/LT3',
                 'key':'AActualPosition.value',
                 'dim':None},
    "AFS_HalfWP": {'source':'SCS_ILH_LAS/MOTOR/ROT_OPA_BWP1',
                 'key':'actualPosition.value',
                 'dim':None},
    "AFS_FocusLens": {'source':'SCS_ILH_LAS/MOTOR/LT_SPARE1',
                 'key':'actualPosition.value',
                 'dim':None},
    # 2nd lens of telescope
    "AFS_TeleLens": {'source':'SCS_ILH_LAS/MOTOR/LT2',
                 'key':'actualPosition.value',
                 'dim':None},

    # PP LASER 800 nm path
    "PP800_PhaseShifter": {'source':'SCS_ILH_LAS/DOOCS/PP800_PHASESHIFTER',
                 'key':'actualPosition.value',
                 'dim':None},
    "PP800_SynchDelayLine": {'source':'SCS_ILH_LAS/DOOCS/PPL_OPT_DELAY',
                 'key':'actualPosition.value',
                 'dim':None},
    "PP800_DelayLine": {'source':'SCS_ILH_LAS/MOTOR/LT3',
                 'key':'AActualPosition.value',
                 'dim':None},
    "PP800_HalfWP": {'source':'SCS_ILH_LAS/MOTOR/ROT8WP1',
                 'key':'actualPosition.value',
                 'dim':None},
    "PP800_FocusLens": {'source':'SCS_ILH_LAS/MOTOR/LT_SPARE1',
                 'key':'actualPosition.value',
                 'dim':None},
    # 1st lens of telescope (setup of August 2019)
    "PP800_TeleLens": {'source':'SCS_ILH_LAS/MOTOR/LT7',
                 'key':'actualPosition.value',
                 'dim':None},
    "ILH_8CAM1": {'source':'SCS_ILH_LAS/CAM/8CAM1:daqOutput',
                'key':'data.image.pixels',
                'dim':['8cam1_y', '8cam1_x']},

    
    # FFT
    "scannerX": {'source':'SCS_CDIFFT_SAM/LMOTOR/SCANNERX',
                 'key':'actualPosition.value',
                 'dim':None},
    "scannerY": {'source':'SCS_CDIFFT_SAM/MOTOR/SCANNERY',
                 'key':'actualPosition.value',
                 'dim':None},
    "scannerY_enc": {'source':'SCS_CDIFFT_SAM/ENC/SCANNERY',
                     'key':'value.value',
                     'dim':None},
    "SAM-Z": {'source':'SCS_CDIFFT_MOV/ENC/SAM_Z',
              'key':'value.value',
              'dim':None},
    "magnet": {'source':'SCS_CDIFFT_MAG/ASENS/CURRENT',
               'key':'value.value',
               'dim':None},
    "magnet_old": {'source':'SCS_CDIFFT_MAG/SUPPLY/CURRENT',
               'key':'actualCurrent.value',
               'dim':None},
    
    "Vertical_FDM": {'source':'SCS_CDIFFT_LDM/CAM/CAMERA1A:daqOutput',
                'key':'data.image.pixels',
                'dim':['vfdm_y', 'vfdm_x']},

    # FastCCD, if in raw folder, raw images
    #          if in proc folder, dark substracted and relative gain corrected
    "fastccd": {'source':'SCS_CDIDET_FCCD2M/DAQ/FCCD:daqOutput',
                'key':'data.image.pixels',
                'dim':['x', 'y']},
    # FastCCD with common mode correction
    "fastccd_cm": {'source':'SCS_CDIDET_FCCD2M/DAQ/FCCD:daqOutput',
                'key':'data.image.pixels_cm',
                'dim':['x', 'y']},
    # FastCCD charge split correction in very low photon count regime
    "fastccd_classified": {'source':'SCS_CDIDET_FCCD2M/DAQ/FCCD:daqOutput',
                'key':'data.image.pixels_classified',
                'dim':['x', 'y']},
    # FastCCD event multiplicity from the charge split correction:
    # 0: no events
    # 100, 101: single events
    # 200-203: charge split into two pixels in four different orientations
    # 300-303: charge split into three pixels in four different orientations
    # 400-403: charge split into four pixels in four different orientations
    # 1000: charge in more than four neighboring pixels. Cannot be produced by a single photon alone.
    "fastccd_patterns": {'source':'SCS_CDIDET_FCCD2M/DAQ/FCCD:daqOutput',
                'key':'data.image.patterns',
                'dim':['x', 'y']},
    # FastCCD gain map, 0 high gain, 1 medium gain, 2 low gain
    "fastccd_gain": {'source':'SCS_CDIDET_FCCD2M/DAQ/FCCD:daqOutput',
                'key':'data.image.gain',
                'dim':['x', 'y']},
    # FastCCD mask, bad pixel map to be ignored if > 0
    "fastccd_mask": {'source':'SCS_CDIDET_FCCD2M/DAQ/FCCD:daqOutput',
                'key':'data.image.mask',
                'dim':['x', 'y']},

    # TIM
    "MCP1apd": {'source':'SCS_UTC1_ADQ/ADC/1:network',
                'key':'digitizers.channel_1_D.apd.pulseIntegral',
                'dim':['apdId']},
    "MCP1raw": {'source':'SCS_UTC1_ADQ/ADC/1:network',
                'key':'digitizers.channel_1_D.raw.samples',
                'dim':['samplesId']},
    "MCP2apd": {'source':'SCS_UTC1_ADQ/ADC/1:network',
                'key':'digitizers.channel_1_C.apd.pulseIntegral',
                'dim':['apdId']},
    "MCP2raw": {'source':'SCS_UTC1_ADQ/ADC/1:network',
                'key':'digitizers.channel_1_C.raw.samples',
                'dim':['samplesId']},
    "MCP3apd": {'source':'SCS_UTC1_ADQ/ADC/1:network',
                'key':'digitizers.channel_1_B.apd.pulseIntegral',
                'dim':['apdId']},
    "MCP3raw": {'source':'SCS_UTC1_ADQ/ADC/1:network',
                'key':'digitizers.channel_1_B.raw.samples',
                'dim':['samplesId']},
    "MCP4apd": {'source':'SCS_UTC1_ADQ/ADC/1:network',
                'key':'digitizers.channel_1_A.apd.pulseIntegral',
                'dim':['apdId']},
    "MCP4raw": {'source':'SCS_UTC1_ADQ/ADC/1:network',
                'key':'digitizers.channel_1_A.raw.samples',
                'dim': ['samplesId']},

    # FastADC
    "FastADC0peaks": {'source':'SCS_UTC1_MCP/ADC/1:channel_0.output',
                'key':'data.peaks',
                'dim':['peakId']},
    "FastADC0raw": {'source':'SCS_UTC1_MCP/ADC/1:channel_0.output',
                'key':'data.rawData',
                'dim':['fadc_samplesId']},
    "FastADC1peaks": {'source':'SCS_UTC1_MCP/ADC/1:channel_1.output',
                'key':'data.peaks',
                'dim':['peakId']},
    "FastADC1raw": {'source':'SCS_UTC1_MCP/ADC/1:channel_1.output',
                'key':'data.rawData',
                'dim':['fadc_samplesId']},
    "FastADC2peaks": {'source':'SCS_UTC1_MCP/ADC/1:channel_2.output',
                'key':'data.peaks',
                'dim':['peakId']},
    "FastADC2raw": {'source':'SCS_UTC1_MCP/ADC/1:channel_2.output',
                'key':'data.rawData',
                'dim':['fadc_samplesId']},
    "FastADC3peaks": {'source':'SCS_UTC1_MCP/ADC/1:channel_3.output',
                'key':'data.peaks',
                'dim':['peakId']},
    "FastADC3raw": {'source':'SCS_UTC1_MCP/ADC/1:channel_3.output',
                'key':'data.rawData',
                'dim':['fadc_samplesId']},
    "FastADC4peaks": {'source':'SCS_UTC1_MCP/ADC/1:channel_4.output',
                'key':'data.peaks',
                'dim':['peakId']},
    "FastADC4raw": {'source':'SCS_UTC1_MCP/ADC/1:channel_4.output',
                'key':'data.rawData',
                'dim':['fadc_samplesId']},
    "FastADC5peaks": {'source':'SCS_UTC1_MCP/ADC/1:channel_5.output',
                'key':'data.peaks',
                'dim':['peakId']},
    "FastADC5raw": {'source':'SCS_UTC1_MCP/ADC/1:channel_5.output',
                'key':'data.rawData',
                'dim':['fadc_samplesId']},
    "FastADC6peaks": {'source':'SCS_UTC1_MCP/ADC/1:channel_6.output',
                'key':'data.peaks',
                'dim':['peakId']},
    "FastADC6raw": {'source':'SCS_UTC1_MCP/ADC/1:channel_6.output',
                'key':'data.rawData',
                'dim':['fadc_samplesId']},
    "FastADC7peaks": {'source':'SCS_UTC1_MCP/ADC/1:channel_7.output',
                'key':'data.peaks',
                'dim':['peakId']},
    "FastADC7raw": {'source':'SCS_UTC1_MCP/ADC/1:channel_7.output',
                'key':'data.rawData',
                'dim':['fadc_samplesId']},
    "FastADC8peaks": {'source':'SCS_UTC1_MCP/ADC/1:channel_8.output',
                'key':'data.peaks',
                'dim':['peakId']},
    "FastADC8raw": {'source':'SCS_UTC1_MCP/ADC/1:channel_8.output',
                'key':'data.rawData',
                'dim':['fadc_samplesId']},
    "FastADC9peaks": {'source':'SCS_UTC1_MCP/ADC/1:channel_9.output',
                'key':'data.peaks',
                'dim':['peakId']},
    "FastADC9raw": {'source':'SCS_UTC1_MCP/ADC/1:channel_9.output',
                'key':'data.rawData',
                'dim':['fadc_samplesId']},

    # KARABACON
    "KARABACON": {'source':'SCS_DAQ_SCAN/MDL/KARABACON',
                    'key': 'actualStep.value',
                    'dim': None},
    
    #GOTTHARD
    "Gotthard1": {'source':'SCS_PAM_XOX/DET/GOTTHARD_RECEIVER1:daqOutput',
                    'key': 'data.adc',
                    'dim': ['gott_pId','pixelId']},
    "Gotthard2": {'source':'SCS_PAM_XOX/DET/GOTTHARD_RECEIVER2:daqOutput',
                    'key': 'data.adc',
                    'dim': ['gott_pId','pixelId']}
}

def extractBunchPattern(bp_table=None, key='sase3', runDir=None):
    ''' generate the bunch pattern and number of pulses of a source directly from the
        bunch pattern table and not using the MDL device BUNCH_DECODER. This is 
        inspired by the euxfel_bunch_pattern package, 
        https://git.xfel.eu/gitlab/karaboDevices/euxfel_bunch_pattern
        Inputs:
            bp_table: DataArray corresponding to the mnemonics "bunchPatternTable".
                      If None, the bunch pattern table is loaded using runDir.
            key: str, ['sase1', 'sase2', 'sase3', 'scs_ppl']
            runDir: karabo_data run directory. Required only if bp_table is None.
            
        Outputs:
            bunchPattern: DataArray containing indices of the sase/laser pulses for 
            each train
            npulses: DataArray containing the number of pulses for each train
                  
    '''
    keys=['sase1', 'sase2', 'sase3', 'scs_ppl']
    if key not in keys:
        raise ValueError(f'Invalid key "{key}", possible values are {keys}')
    if bp_table is None:
        if runDir is None:
            raise ValueError('bp_table and runDir cannot both be None')
        bp_mnemo = mnemonics['bunchPatternTable']
        if bp_mnemo['source'] not in runDir.all_sources:
            raise ValueError('Source {} not found in run'.format(
                                mnemonics['bunchPatternTable']['source']))
        else:
            bp_table = runDir.get_array(bp_mnemo['source'],bp_mnemo['key'], 
                                        extra_dims=bp_mnemo['dim'])
    # define relevant masks, see euxfel_bunch_pattern package for details
    DESTINATION_MASK = 0xf << 18
    DESTINATION_T4D = 4 << 18   # SASE1/3 dump
    DESTINATION_T5D = 2 << 18  # SASE2 dump
    PHOTON_LINE_DEFLECTION = 1 << 27  # Soft kick (e.g. SA3)
    LASER_SEED6 = 1 << 13
    if 'sase' in key:
        sase = int(key[4])
        destination = DESTINATION_T5D if (sase == 2) else DESTINATION_T4D
        matched = (bp_table & DESTINATION_MASK) == destination
        if sase == 1:
            # Pulses to SASE 1 when soft kick is off
            matched &= (bp_table & PHOTON_LINE_DEFLECTION) == 0
        elif sase == 3:
            # Pulses to SASE 3 when soft kick is on
            matched &= (bp_table & PHOTON_LINE_DEFLECTION) != 0
    elif key=='scs_ppl':
        matched = (bp_table & LASER_SEED6) != 0
    
    # create table of indices where bunch pattern and mask match
    nz = np.nonzero(matched.values)
    dim_pId = matched.shape[1]
    bunchPattern = np.ones(matched.shape, dtype=np.uint64)*dim_pId
    bunchPattern[nz] = nz[1]
    bunchPattern = np.sort(bunchPattern)
    npulses = np.count_nonzero(bunchPattern<dim_pId, axis=1)
    bunchPattern[bunchPattern == dim_pId] = 0

    bunchPattern = xr.DataArray(bunchPattern[:,:1000], dims=['trainId', 'bunchId'],
                          coords={'trainId':matched.trainId}, 
                          name=key)
    npulses = xr.DataArray(npulses, dims=['trainId'],
                                coords={'trainId':matched.trainId}, 
                                name=f'npulses_{key}')
    return bunchPattern, npulses

def load(fields, runNB, proposalNB, subFolder='raw', display=False, validate=False,
         subset=by_index[:], rois={}, useBPTable=True):
    """ Load a run and extract the data. Output is an xarray with aligned trainIds

        Inputs:
            fields: list of mnemonic strings to load specific data such as "fastccd", "SCS_XGM",
                or dictionnaries defining a custom mnemonic such as
                {"extra": {'SCS_CDIFFT_MAG/SUPPLY/CURRENT', 'actual_current.value', None}}
            runNB: (str, int) run number as integer
            proposalNB: (str, int) of the proposal number e.g. 'p002252' or 2252
            subFolder: (str) sub-folder from which to load the data. Use 'raw' for raw
                data or 'proc' for processed data.
            display: (bool) whether to show the run.info or not
            validate: (bool) whether to run karabo-data-validate or not
            subset: a subset of train that can be load with by_index[:5] for the
                first 5 trains
            rois: a dictionnary of mnemonics with a list of rois definition and the desired
                names, for example {'fastccd':{'ref':{'roi':by_index[730:890, 535:720],
                'dim': ['ref_x', 'ref_y']}, 'sam':{'roi':by_index[1050:1210, 535:720],
                'dim': ['sam_x', 'sam_y']}}}
            useBPTable: If True, uses the raw bunch pattern table to extract sase pulse
                number and indices in the trains. If false, load the data from BUNCH_DECODER
                middle layer device.

        Outputs:
            res: an xarray DataSet with aligned trainIds
    """
    
    if isinstance(runNB, int):
        runNB = 'r{:04d}'.format(runNB)
    if isinstance(proposalNB,int):
        proposalNB = 'p{:06d}'.format(proposalNB)
    runFolder = os.path.join(find_proposal(proposalNB), subFolder, runNB)
    run = RunDirectory(runFolder).select_trains(subset)

    if validate:
        get_ipython().system('karabo-data-validate ' + runFolder)
    if display:
        print('Loading data from {}'.format(runFolder))
        run.info()

    keys = []
    vals = []

    # load pulse pattern info
    if useBPTable:
        bp_mnemo = mnemonics['bunchPatternTable']
        if bp_mnemo['source'] not in run.all_sources:
            print('Source {} not found in run. Skipping!'.format(
                                mnemonics['bunchPatternTable']['source']))
        else:
            bp_table = run.get_array(bp_mnemo['source'],bp_mnemo['key'], 
                                        extra_dims=bp_mnemo['dim'])
            sase1, npulses_sase1 = extractBunchPattern(bp_table, 'sase1')
            sase3, npulses_sase3 = extractBunchPattern(bp_table, 'sase3')
            keys += ["sase1", "npulses_sase1", "sase3", "npulses_sase3"]
            vals += [sase1, npulses_sase1, sase3, npulses_sase3]
    else:
        fields += ["sase1", "sase3", "npulses_sase3", "npulses_sase1"]

    for f in fields:

        if type(f) == dict:
            # extracting mnemomic defined on the spot
            if len(f.keys()) > 1:
                print('Loading only one "on-the-spot" mnemonic at a time, skipping all others !')
            k = list(f.keys())[0]
            v = f[k]
        else:
            # extracting mnemomic from the table
            if f in mnemonics:
                v = mnemonics[f]
                k = f
            else:
                print('Unknow mnemonic "{}". Skipping!'.format(f))
                continue
                
        if k in keys:
            continue # already loaded, skip

        if display:
            print('Loading {}'.format(k))

        if v['source'] not in run.all_sources:
            print('Source {} not found in run. Skipping!'.format(v['source']))
            continue

        if k not in rois:
            # no ROIs selection, we read everything
            vals.append(run.get_array(v['source'], v['key'], extra_dims=v['dim']))
            keys.append(k)
        else:
            # ROIs selection, for each ROI we select a region of the data and save it with new name and dimensions
            for nk,nv in rois[k].items():
                vals.append(run.get_array(v['source'], v['key'], extra_dims=nv['dim'], roi=nv['roi']))
                keys.append(nk)
    
    aligned_vals = xr.align(*vals, join='inner')
    result = dict(zip(keys, aligned_vals))
    result = xr.Dataset(result)
    result.attrs['run'] = run
    result.attrs['runFolder'] = runFolder
    return result

def concatenateRuns(runs):
    """ Sorts and concatenate a list of runs with identical data variables along the
        trainId dimension.
        
        Input:
            runs: (list) the xarray Datasets to concatenate
        Output:
            a concatenated xarray Dataset
    """
    firstTid = {i: int(run.trainId[0].values) for i,run in enumerate(runs)}
    orderedDict = dict(sorted(firstTid.items(), key=lambda t: t[1]))
    orderedRuns = [runs[i] for i in orderedDict]
    keys = orderedRuns[0].keys()
    for run in orderedRuns[1:]:
        if run.keys() != keys:
            print('data fields between different runs are not identical. Cannot combine runs.')
            return
    
    result = xr.concat(orderedRuns, dim='trainId')
    result.attrs['run'] = [run.attrs['run'] for run in orderedRuns]
    result.attrs['runFolder'] = [run.attrs['runFolder'] for run in orderedRuns]
    return result