From 3ea78f345aac4c8e41aba867c8e2bc099f33421e Mon Sep 17 00:00:00 2001
From: Steffen Hauf <haufs@max-exfl015.desy.de>
Date: Fri, 22 Jun 2018 11:38:18 +0200
Subject: [PATCH] Reorganization
---
.gitignore | 5 +++--
batch/automode_start_agipd.sh | 6 +++---
batch/automode_start_lpd.sh | 2 +-
setup.py | 10 ++++++----
xfel_calibrate/automode.py | 2 +-
xfel_calibrate/calibrate.py | 10 +++++-----
xfel_calibrate/notebooks.py | 20 +++++++++++---------
7 files changed, 30 insertions(+), 25 deletions(-)
diff --git a/.gitignore b/.gitignore
index 1ccd5e15a..f0359b27e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,6 @@
*/*.out
-*/.ipynb_checkpoints
-*/.ipynb_checkpoints/*
+notebooks/*/.ipynb_checkpoints
+notebooks/*/.ipynb_checkpoints/*
*.gif
*.tmp
*.png
@@ -23,3 +23,4 @@ Test
*.out
slurm_tmp*
./temp
+*egg*
diff --git a/batch/automode_start_agipd.sh b/batch/automode_start_agipd.sh
index bf9e59fcd..d71801846 100755
--- a/batch/automode_start_agipd.sh
+++ b/batch/automode_start_agipd.sh
@@ -1,5 +1,5 @@
#!/bin/bash
#module load anaconda/3
-source /gpfs/exfel/data/user/haufs/karabo/activate
-cd /gpfs/exfel/data/user/haufs/scripted_offline_cal/pycalibration/
-python /gpfs/exfel/data/user/haufs/scripted_offline_cal/pycalibration/automode.py --input /gpfs/exfel/exp/SPB/201830/p900022/raw --output /gpfs/exfel/exp/SPB/201830/p900022/proc/ --base-cal-store /gpfs/exfel/exp/SPB/201701/p002012/proc/calibration/agipd_base_store_64_r0494_r0495.h5 --offset-cal-store /gpfs/exfel/exp/SPB/201701/p002012/proc/calibration/dark/agipd_offset_store --mem-cells 64 --detector AGIPD --type correct --partition exfel
+source /gpfs/exfel/data/scratch/haufs/clean_cal/karabo/activate
+cd /gpfs/exfel/data/scratch/haufs/clean_cal/pycalibration/
+python /gpfs/exfel/data/scratch/haufs/clean_cal/pycalibration/automode.py --input /gpfs/exfel/exp/SPB/201701/p002038/raw --output /gpfs/exfel/exp/SPB/201701/p002038/proc/ --base-cal-store /gpfs/exfel/exp/SPB/201830/p900019/proc/calibration0618/Merged/agipd_base_store_64_r0820.h5 --offset-cal-store /gpfs/exfel/exp/SPB/201830/p900019/proc/calibration0618/dark1MHZ/agipd_offset_store_r0887_r0888_r0889.h5 --mem-cells 128 --detector AGIPD --type correct --partition exfel --only-new --start-time 20180613
diff --git a/batch/automode_start_lpd.sh b/batch/automode_start_lpd.sh
index 6e226a51c..a1f0402af 100755
--- a/batch/automode_start_lpd.sh
+++ b/batch/automode_start_lpd.sh
@@ -2,5 +2,5 @@
#module load anaconda/3
source /gpfs/exfel/data/scratch/haufs/clean_cal/karabo/activate
cd /gpfs/exfel/data/scratch/haufs/clean_cal/pycalibration
-python3 automode_nbc.py --input /gpfs/exfel/exp/FXE/201701/p002052/raw/ --output /gpfs/exfel/exp/FXE/201701/p002052/proc/ --base-cal-store /gpfs/exfel/d/proc/FXE/201830/p900020/calibration/lpd_ci_store_CHANID_16_5pf.h5 --offset-cal-store /gpfs/exfel/exp/FXE/201701/p002052/proc/calibration/dark/lpd_offset_store_ --mem-cells 128 --detector LPD --type correct --partition exfel --ff-cal-store /gpfs/exfel/exp/FXE/201830/p900020/proc/calibration/flat_field/lpd_flatfield_store_r0077.h5 --only-new --start-time 2018-06-08
+python3 automode.py --input /gpfs/exfel/exp/FXE/201701/p002045/raw/ --output /gpfs/exfel/exp/FXE/201701/p002045/proc/ --base-cal-store /gpfs/exfel/exp/FXE/201830/p900020/proc/calibration/CI5pf_pp/lpd_ci_store_CHANID_16_5pf.h5 --offset-cal-store /gpfs/exfel/exp/FXE/201701/p002045/proc/calibration0618/dark/lpd_offset_store_r0279_r0280_r0281_5pf.h5 --mem-cells 128 --detector LPD --type correct --partition exfel --ff-cal-store /gpfs/exfel/exp/FXE/201830/p900020/proc/calibration/flat_field_ci_pp/lpd_flatfield_store_r0230.h5 --only-new --start-time 2018-06-12
#r0288,r0289,r0290,r0291,r0292,r0293,r0294,r0295,r0296,r0297,r0298,r0299,r0300,r0301,r0302 --no-ff
diff --git a/setup.py b/setup.py
index e9dcaea6b..3e7a058f7 100644
--- a/setup.py
+++ b/setup.py
@@ -26,9 +26,11 @@ class PostInstallCommand(install):
setup(
name='European XFEL Offline Calibration',
version="1.0",
- packages=['cal_tools', 'calibrate'],
- package_dir={'cal_tools': 'cal_tools/cal_tools',
- 'calibrate': './'},
+ packages=['cal_tools', 'xfel_calibrate'],
+ package_dir={'cal_tools': 'cal_tools',
+ 'xfel_calibrate': './'},
+ package_data={'xfel_calibrate/notebooks': 'notebooks',
+ 'xfel_calibrate/bin': 'bin'},
cmdclass={
'install': PostInstallCommand,
},
@@ -39,7 +41,7 @@ setup(
description='',
entry_points = {
'console_scripts': [
- 'xfel-calibrate = calibrate:run',
+ 'xfel-calibrate = xfel_calibrate.calibrate:run',
],
},
diff --git a/xfel_calibrate/automode.py b/xfel_calibrate/automode.py
index d5ab3dccb..71856cd88 100644
--- a/xfel_calibrate/automode.py
+++ b/xfel_calibrate/automode.py
@@ -57,7 +57,7 @@ def run():
#srun_base = ["sbatch", "-p", partition, "-t", "16:00:00", '--mem', '400G']
basekey = "--base-store" if det.upper() == "AGIPD" else "--relgain-store"
srun_base = ["python3",
- os.path.abspath("{}/calibrate.py".format(os.getcwd())),
+ os.path.abspath("{}/calibrate.py".format(os.path.realpath(__file__))),
det.upper(),
"CORRECT",
basekey, base_cal_store,
diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py
index 3036188d2..4f9f3e6a6 100755
--- a/xfel_calibrate/calibrate.py
+++ b/xfel_calibrate/calibrate.py
@@ -14,8 +14,8 @@ from subprocess import Popen, PIPE, check_output
import sys
from uuid import uuid4
import warnings
-from settings import *
-from notebooks import notebooks
+from .settings import *
+from .notebooks import notebooks
# Add a class combining raw description formatting with
@@ -161,7 +161,7 @@ if len(sys.argv) == 3 and "-h" in sys.argv[2]:
# The information is extracted from the first markdown cell of
# the notebook.
for caltype, notebook in det_notebooks.items():
- with open(notebook["notebook"], "r") as f:
+ with open(os.path.abspath(notebook["notebook"]), "r") as f:
nb = nbformat.read(f, as_version=4)
msg += make_epilog(nb, caltype=caltype)
@@ -176,7 +176,7 @@ elif len(sys.argv) >= 3:
detector = sys.argv[1].upper()
caltype = sys.argv[2].upper()
try:
- notebook = notebooks[detector][caltype]["notebook"]
+ notebook = os.path.abspath(notebooks[detector][caltype]["notebook"])
cvar = notebooks[detector][caltype].get("concurrency",
{"parameter": None,
"default concurrency": None,
@@ -271,7 +271,7 @@ def concurrent_run(temp_path, nb, nbname, args, cparm=None, cval=None,
srun_base = []
- srun_base += [os.path.abspath("{}/slurm_calibrate.sh".format(os.getcwd())), # path to helper sh
+ srun_base += [os.path.abspath("{}/../../bin/slurm_calibrate.sh".format(os.path.realpath(__file__))), # path to helper sh
os.path.abspath(nbpath), # path to notebook
python_path, # path to python
ipython_path, # path to ipython
diff --git a/xfel_calibrate/notebooks.py b/xfel_calibrate/notebooks.py
index 421f675fb..8ef1d0d06 100644
--- a/xfel_calibrate/notebooks.py
+++ b/xfel_calibrate/notebooks.py
@@ -1,31 +1,33 @@
+# paths are relative to the location of setup.py in the main package
+
notebooks = {
"AGIPD": {
"DARK": {
- "notebook": "AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb",
+ "notebook": "notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb",
"concurrency": {"parameter": None,
"default concurrency": None,
"cluster cores": 8},
},
"PC": {
- "notebook": "AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb",
+ "notebook": "notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb",
"concurrency": {"parameter": "modules",
"default concurrency": 16,
"cluster cores": 32},
},
"FF": {
- "notebook": "AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb",
+ "notebook": "notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb",
"concurrency": {"parameter": "modules",
"default concurrency": 16,
"cluster cores": 16},
},
"CORRECT": {
- "notebook": "AGIPD/AGIPD_Correct_and_Verify.ipynb",
+ "notebook": "notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb",
"concurrency": {"parameter": None,
"default concurrency": None,
"cluster cores": 32},
},
"COMBINE": {
- "notebook": "AGIPD/AGIPD_Characterize_Gain_Combine_NBC.ipynb",
+ "notebook": "notebooks/AGIPD/AGIPD_Characterize_Gain_Combine_NBC.ipynb",
"concurrency": {"parameter": None,
"default concurrency": None,
"cluster cores": 8},
@@ -33,25 +35,25 @@ notebooks = {
},
"LPD": {
"DARK": {
- "notebook": "LPD/LPDChar_Darks_NBC.ipynb",
+ "notebook": "notebooks/LPD/LPDChar_Darks_NBC.ipynb",
"concurrency": {"parameter": None,
"default concurrency": None,
"cluster cores": 8},
},
"PC": {
- "notebook": "LPD/Characterize_LPD_GAIN_CI_per_pixel_NBC.ipynb",
+ "notebook": "notebooks/LPD/Characterize_LPD_GAIN_CI_per_pixel_NBC.ipynb",
"concurrency": {"parameter": "modules",
"default concurrency": 16,
"cluster cores": 32},
},
"FF": {
- "notebook": "LPD/LPD_FlatField_Radial_per_pixel_CI_NBC.ipynb",
+ "notebook": "notebooks/LPD/LPD_FlatField_Radial_per_pixel_CI_NBC.ipynb",
"concurrency": {"parameter": None,
"default concurrency": None,
"cluster cores": 8},
},
"CORRECT": {
- "notebook": "LPD/LPD_Correct_and_Verify.ipynb",
+ "notebook": "notebooks/LPD/LPD_Correct_and_Verify.ipynb",
"concurrency": {"parameter": None,
"default concurrency": None,
"cluster cores": 32},
--
GitLab