diff --git a/batch/automode_start_agipd.sh b/batch/automode_start_agipd.sh
deleted file mode 100755
index d71801846d654e6156a3a403386ffee5f89c4edf..0000000000000000000000000000000000000000
--- a/batch/automode_start_agipd.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-#module load anaconda/3
-source /gpfs/exfel/data/scratch/haufs/clean_cal/karabo/activate
-cd /gpfs/exfel/data/scratch/haufs/clean_cal/pycalibration/
-python /gpfs/exfel/data/scratch/haufs/clean_cal/pycalibration/automode.py --input /gpfs/exfel/exp/SPB/201701/p002038/raw --output /gpfs/exfel/exp/SPB/201701/p002038/proc/ --base-cal-store /gpfs/exfel/exp/SPB/201830/p900019/proc/calibration0618/Merged/agipd_base_store_64_r0820.h5 --offset-cal-store /gpfs/exfel/exp/SPB/201830/p900019/proc/calibration0618/dark1MHZ/agipd_offset_store_r0887_r0888_r0889.h5 --mem-cells 128 --detector AGIPD --type correct --partition exfel --only-new --start-time 20180613
diff --git a/batch/automode_start_lpd.sh b/batch/automode_start_lpd.sh
deleted file mode 100755
index 5f66b7e2ed44a9fe0b62577c096b9d3070aff35f..0000000000000000000000000000000000000000
--- a/batch/automode_start_lpd.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/bash
-#module load anaconda/3
-source /gpfs/exfel/data/scratch/haufs/clean_cal/karabo/activate
-cd /gpfs/exfel/data/scratch/haufs/clean_cal/pycalibration/xfel_calibrate
-python3 automode.py --input /gpfs/exfel/exp/FXE/201801/p002092/raw/\
- --output /gpfs/exfel/exp/FXE/201801/p002092/proc/\
- --mem-cells 512\
- --calfile /gpfs/exfel/exp/FXE/201801/p002092/proc/cal_constants.h5\
- --detector LPD\
- --type correct\
- --partition exfel\
- --only-new\
- --start-time 2018-08-29
diff --git a/bin/automode.sh b/bin/automode.sh
deleted file mode 100755
index c3e9fb091f46d99fea1a57d729c958090078b474..0000000000000000000000000000000000000000
--- a/bin/automode.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/bash
-#SBATCH -N 1 # number of nodes
-#SBATCH -n 32 # number of cores
-#module load python3
-#~/.local/bin/ipcluster start --n=32 --daemon --ip="*"
-source /gpfs/exfel/data/scratch/haufs/clean_cal/karabo/activate
-ipython profile create ${11} --parallel
-/gpfs/exfel/data/user/haufs/karabo/extern/bin/ipcluster start --n=8 --profile=${11} --daemon &
-sleep 30
-/gpfs/exfel/data/user/haufs/karabo/extern/bin/python "$@"
-/gpfs/exfel/data/user/haufs/karabo/extern/bin/ipcluster stop --profile=${11}
-rm -rf "/home/haufs/.ipython/profile_${11}"
-#echo ${11}
diff --git a/docs/source/advanced.rst b/docs/source/advanced.rst
index b193634db1d1a717693daa558287258e14ad112c..9f623ab8cc46ef589babef4d044c72341c43dc4c 100644
--- a/docs/source/advanced.rst
+++ b/docs/source/advanced.rst
@@ -31,76 +31,6 @@ The path to data files is defined from script parameters. A typical data path, w
Where `MID` is an instrument name, `201930` is a cycle, `900071` is a proposal number.
-Initiating Automated Offline Correction
----------------------------------------
-
-In order to initiate the automatic offline correction for a given detector and instrument
-navigate to the root path of the European XFEL Offline Calibration tool chain.
-
-It is best to run the following as a SLURM batch job in itself, as this insures it is
-independent of the terminal you are on an will persistently run for a given time period:
-
-1. do a trial run of `automode_nbc.py` to test your settings::
-
- python3 automode.py --input /gpfs/exfel/exp/FXE/201701/p002052/raw/ \
- --output /gpfs/exfel/exp/FXE/201701/p002052/proc/ \
- --base-cal-store /gpfs/.../lpd_ci_store_CHANID_16_5pf.h5 \
- --offset-cal-store /gpfs/.../lpd_offset_store_r0279_r0280_r0281_5pf.h5 \
- --mem-cells 128 \
- --detector LPD \
- --type correct \
- --partition exfel \
- --ff-cal-store /gpfs/.../lpd_flatfield_store_r0230.h5 \
- --only-new \
- --start-time 2018-06-08
-
-Here `input` should refer to the proposal `raw` path, `output` to the proposal `proc`.
-The cal_stores refer to constants from files, usually they will be overwritten by
-usage of the calibration database, but a fallback is currently required. Note the `CHANID`
-field in the base_store constant. It signifies that module numbers are to be inserted
-there. Finally, `only-new` should be used in combination with `start-time` to if previous
-data in the proposal was calibrated and should be left untouched. For fresh proposals
-these can be omitted.
-
-For testing you can also omit `only-new` and `start-time` and instead use the `--run XXX`
-or `--run XXX-YYY` option to trial run on a specific set of runs (possibly into a temporary
-directory). Use the `--overwrite` option to do this multiple times.
-
-Once you are satisfied with the results, place everthing in a bash script which will
-initiate the actual startup::
-
- nano automode_startup.sh
-
-In this script make sure that a python environment with the necessary tools is available.
-Straight-forward this could be the environment you've installed the tool-chain in. The
-script should thus look similar to this::
-
- #!/bin/bash
- source /gpfs/path/to/karabo/activate
- cd /gpfs/path/to/pycalibration
-
- python3 automode.py --input /gpfs/exfel/exp/FXE/201701/p002052/raw/ \
- --output /gpfs/exfel/exp/FXE/201701/p002052/proc/ \
- --base-cal-store /gpfs/.../lpd_ci_store_CHANID_16_5pf.h5 \
- --offset-cal-store /gpfs/.../lpd_offset_store_r0279_r0280_r0281_5pf.h5 \
- --mem-cells 128 \
- --detector LPD \
- --type correct \
- --partition exfel \
- --ff-cal-store /gpfs/.../lpd_flatfield_store_r0230.h5 \
- --only-new \
- --start-time 2018-06-08
-
-Now launch the script as a slurm job::
-
- sbatch -p exfel -t 178:00:00 --mail-type=ALL automode_startup.sh
-
-Make sure to give the `-t` option and set it to cover the expected beam-time duration.
-Otherwise the SLURM job will be cancelled after an hour. The `--mail-type` option will
-result in an email being sent to you (or rather the user currently logged in) in case
-of any failures, etc.
-
-
Extending Correction Notebooks on User Request
----------------------------------------------
diff --git a/xfel_calibrate/automode.py b/xfel_calibrate/automode.py
deleted file mode 100644
index b1c817524b57745bfa12dcf4b18868046d6199ed..0000000000000000000000000000000000000000
--- a/xfel_calibrate/automode.py
+++ /dev/null
@@ -1,125 +0,0 @@
-import argparse
-import copy
-from dateutil import parser as dtparser
-import glob
-import os
-from subprocess import Popen, PIPE
-from time import sleep
-import datetime
-import time
-from uuid import uuid4
-
-parser = argparse.ArgumentParser(description="Main entry point "
- "for offline calibration")
-parser.add_argument("--input", type=str)
-parser.add_argument("--output", type=str)
-parser.add_argument("--calfile", type=str)
-parser.add_argument("--mem-cells", type=str)
-parser.add_argument("--detector", type=str)
-parser.add_argument("--type", type=str)
-parser.add_argument("--partition", type=str, default="upex")
-parser.add_argument("--no-relgain", action="store_true", default=False)
-parser.add_argument("--overwrite", action="store_true", default=False)
-parser.add_argument("--no-ff", action="store_true", default=False)
-parser.add_argument("--runs", type=str, default="all")
-parser.add_argument("--only-new", action="store_true", default=False)
-parser.add_argument("--raw-version", type=str, default="2")
-parser.add_argument("--seqs", type=str, default="ALL")
-parser.add_argument("--start-time", type=str, default="now")
-
-LAST_MOD_DELAY = 120 # seconds
-
-def run():
- args = vars(parser.parse_args())
-
- inpath = os.path.abspath(args["input"])
- output = os.path.abspath(args["output"])
- if not os.path.exists(output):
- os.makedirs(output)
- mem_cells = str(args["mem_cells"])
- partition = args["partition"]
- no_relgain = bool(args["no_relgain"])
- det = args["detector"].upper()
- overwrite = bool(args["overwrite"])
- raw_version = args["raw_version"]
- seqs = args["seqs"]
- start_time = args["start_time"]
- calfile = args["calfile"]
- no_ff = False
-
- #srun_base = ["sbatch", "-p", partition, "-t", "16:00:00", '--mem', '400G']
- basekey = "--base-store" if det.upper() == "AGIPD" else "--relgain-store"
- srun_base = ["xfel-calibrate", #"python3",
-# os.path.abspath("{}/calibrate.py".format(os.path.dirname(os.path.realpath(__file__)))),
- det.upper(),
- "CORRECT",
- "--calfile", calfile,
- "--mem-cells", mem_cells,
- ]
-
- runs = args["runs"]
- if runs.upper() == "ALL":
- runs = runs.upper()
- else:
- lruns = []
- for rcomp in runs.split(","):
- if "-" in rcomp:
- start, end = rcomp.split("-")
- lruns += list(range(int(start.replace("r", "")), int(end.replace("r", ""))))
- else:
- lruns += [int(rcomp.replace("r", ""))]
- lruns = ["r{:04d}".format(r) for r in lruns]
- runs = (",".join(lruns))
- print(runs)
- only_new = bool(args["only_new"])
- print("Checking only new runs: {}".format(only_new))
- pending = []
- first_loop = True
- if start_time.upper() == "NOW":
- start_time = time.time()
- else:
- start_time = dtparser.parse(start_time)
- if only_new:
- print("Only treating runs newer than: {}".format(start_time.isoformat()))
- while True:
- idirs = set(os.listdir(inpath))
- odirs = set(os.listdir(output))
- to_process = (idirs - odirs) if not overwrite else idirs
- for dir in to_process:
- if dir in pending or not dir:
- continue
- if runs != "ALL" and dir not in runs:
- continue
- # check if the folder is older than
- # a given delay, if not data might still
- # be transferred into it
- now = time.time()
- stat = os.stat("{}/{}".format(inpath, dir))
- last_mod = stat.st_mtime
- if now-last_mod < LAST_MOD_DELAY:
- continue
- if only_new and last_mod < start_time.timestamp():
- print("Skipping {} -> {}".format(dir, datetime.date.fromtimestamp(last_mod).isoformat()))
- pending.append(dir)
- continue
- print("DIR: ", dir)
- s_run = copy.copy(srun_base)
- s_run += ["--in-folder", "{}".format(inpath)]
- s_run += ["--out-folder", output]
- s_run += ["--run", dir[1:]]
- if no_relgain:
- s_run.append("--no-relative-gain")
- if overwrite:
- s_run.append("--overwrite")
-
- if not first_loop or not only_new:
- print(" ".join(s_run))
- Popen(s_run)
- pending.append(dir)
- first_loop = False
- sleep(10)
-
-if __name__ == "__main__":
- run()
-
-
diff --git a/xfel_calibrate/check_proc_files.py b/xfel_calibrate/check_proc_files.py
deleted file mode 100644
index fcf1703256d6721122abf88151687eeadf40d3c6..0000000000000000000000000000000000000000
--- a/xfel_calibrate/check_proc_files.py
+++ /dev/null
@@ -1,27 +0,0 @@
-import os
-import sys
-import glob
-
-
-rdir = sys.argv[1]
-pdir = sys.argv[2]
-
-rdirs = glob.glob(rdir+'r*')
-pdirs = glob.glob(pdir+'r*')
-
-rdirs = sorted(rdirs)
-pdirs = sorted(pdirs)
-
-print('Number of raw runs: ', len(rdirs))
-print('Number of processed runs: ', len(pdirs))
-
-for run in pdirs:
- run2 = os.path.basename(run)
-
- rfiles = glob.glob(rdir+run2+'/*AGIPD*.h5')
- pfiles = glob.glob(pdir+run2+'/*AGIPD*.h5')
-
- nraw = len(rfiles)
- nproc = len(pfiles)
-
- print(run2,' ', nraw, ' ', nproc)
diff --git a/xfel_calibrate/notebooks.py b/xfel_calibrate/notebooks.py
index 82b1f559b29b2f0b79bd87369e72a27c34f0ad11..ff69d543511fce0a84db98a0e771ba0fc7747c9e 100644
--- a/xfel_calibrate/notebooks.py
+++ b/xfel_calibrate/notebooks.py
@@ -34,13 +34,6 @@ notebooks = {
"default concurrency": None,
"cluster cores": 8},
},
- "STATS_FROM_DB": {
- "notebook": "notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb",
- "dep_notebooks": ["notebooks/generic/PlotFromCalDB_Summary_NBC.ipynb"],
- "concurrency": {"parameter": "modules",
- "default concurrency": None,
- "cluster cores": 1},
- },
},
"LPD": {
"DARK": {
@@ -76,13 +69,6 @@ notebooks = {
"default concurrency": [-1],
"cluster cores": 2},
},
- "STATS_FROM_DB": {
- "notebook": "notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb",
- "dep_notebooks": ["notebooks/generic/PlotFromCalDB_Summary_NBC.ipynb"],
- "concurrency": {"parameter": "modules",
- "default concurrency": None,
- "cluster cores": 1},
- },
},
"PNCCD": {
"DARK": {
@@ -143,12 +129,6 @@ notebooks = {
"use function": "balance_sequences",
"cluster cores": 4},
},
- "STATS_FROM_DB": {
- "notebook": "notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb",
- "concurrency": {"parameter": None,
- "default concurrency": None,
- "cluster cores": 1},
- },
},
"JUNGFRAU": {
"DARK": {
@@ -164,13 +144,6 @@ notebooks = {
"use function": "balance_sequences",
"cluster cores": 14},
},
-
- "STATS_FROM_DB": {
- "notebook": "notebooks/Jungfrau/PlotFromCalDB_Jungfrau_NBC.ipynb",
- "concurrency": {"parameter": None,
- "default concurrency": None,
- "cluster cores": 1},
- },
},
"EPIX": {
"DARK": {
@@ -188,12 +161,6 @@ notebooks = {
"use function": "balance_sequences",
"cluster cores": 4},
},
- "STATS_FROM_DB": {
- "notebook": "notebooks/ePix/PlotFromCalDB_ePix100_NBC.ipynb",
- "concurrency": {"parameter": None,
- "default concurrency": None,
- "cluster cores": 1},
- },
},
"EPIX10K": {
"DARK": {