From 3eeafbedbe14e14f1167b7875a11c0f21c703a2f Mon Sep 17 00:00:00 2001
From: Karim Ahmed <karim.ahmed@xfel.eu>
Date: Mon, 7 Dec 2020 10:23:21 +0100
Subject: [PATCH] Fix/remove unused influx db
---
bin/cal_influx_feeder.sh | 9 --
bin/slurm_calibrate.sh | 4 -
cal_tools/cal_tools/influx.py | 108 ------------------
docs/source/workflow.rst | 28 +----
.../AGIPD_Characterize_Gain_Combine_NBC.ipynb | 1 -
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 11 +-
.../AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb | 14 +--
.../playground/AGIPD_SingleM_test_Dark.ipynb | 11 +-
...aracterize_AGIPD_Gain_FlatFields_NBC.ipynb | 11 +-
.../Chracterize_AGIPD_Gain_PC_mlearn.ipynb | 12 +-
.../DSSC/Characterize_DSSC_Darks_NBC.ipynb | 8 --
...aracterize_LPD_GAIN_CI_per_pixel_NBC.ipynb | 6 -
notebooks/LPD/LPDChar_Darks_NBC.ipynb | 11 +-
notebooks/LPD/LPD_Correct_and_Verify.ipynb | 46 +-------
...PD_FlatField_Radial_per_pixel_CI_NBC.ipynb | 6 -
requirements.txt | 1 -
16 files changed, 13 insertions(+), 274 deletions(-)
delete mode 100755 bin/cal_influx_feeder.sh
delete mode 100644 cal_tools/cal_tools/influx.py
diff --git a/bin/cal_influx_feeder.sh b/bin/cal_influx_feeder.sh
deleted file mode 100755
index e2d079467..000000000
--- a/bin/cal_influx_feeder.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-notebook=$1
-detector=$2
-caltype=$3
-host_name=`hostname`
-while true; do
- curl -i -XPOST 'http://exflqr18318:8086/write?db=calstats' --data-binary "characterize_load,host=$host_name,detector=$detector,cal_type=$caltype cpu=`cat /proc/loadavg | cut -f1 -d' '`,mem_used=`free | tail -2 | head -1 | tr -s ' ' | cut -d' ' -f3`,mem_free=`free | tail -2 | head -1 | tr -s ' ' | cut -d' ' -f4`"
- sleep 10
-done
diff --git a/bin/slurm_calibrate.sh b/bin/slurm_calibrate.sh
index 3ceb22f4d..05a04cba4 100755
--- a/bin/slurm_calibrate.sh
+++ b/bin/slurm_calibrate.sh
@@ -30,9 +30,6 @@ source /etc/profile.d/modules.sh
module load anaconda/3
module load texlive/2019
-echo "Starting influx feeder"
-./cal_influx_feeder.sh $notebook $detector $caltype > /dev/null 2>&1 &
-
# make sure we use agg backend
export MPLBACKEND=AGG
@@ -62,4 +59,3 @@ if [ "${final}" == "FINAL" ]
then
${python_path} ${finalize} $SLURM_JOB_ID
fi
-killall -9 cal_influx_feeder.sh || true
diff --git a/cal_tools/cal_tools/influx.py b/cal_tools/cal_tools/influx.py
deleted file mode 100644
index 04e1817e6..000000000
--- a/cal_tools/cal_tools/influx.py
+++ /dev/null
@@ -1,108 +0,0 @@
-from datetime import datetime
-from enum import Enum
-import socket
-
-from influxdb import InfluxDBClient
-
-
-class ConstantEntry(Enum):
-
- LOAD = "load_constants"
- SAVE = "save_constants"
-
-
-class InfluxLogger:
-
- def __init__(self, detector=None, instrument=None, notebook=None,
- run=None, proposal=None, mem_cells=None):
- self.detector = detector
- self.instrument = instrument
- self.notebook = notebook
- self.run = run
- self.proposal = proposal
- self.mem_cells = mem_cells
- self.entries = []
-
- def constants_entry(self, optype, qm, constant, success, origin, loadtime, reason=""):
- self.entries.append( {
- "measurement": optype.value,
- "tags": {
- "host": socket.gethostname(),
- "run": self.run,
- "proposal": self.proposal,
- "mem_cells": self.mem_cells,
- "constant": constant,
- "module": qm,
- "origin": origin,
- "detector": self.detector,
- "instrument": self.instrument,
- },
- "time": datetime.utcnow().isoformat(),
- "fields": {
- "success": success,
- "reason": reason,
- "load_time": loadtime,
- }
- })
-
-
- def runtime_entry(self, qm, sequence, filesize, chunksize,
- total_sequences, sequences_per_module,
- success, runtime, reason=""):
- self.entries.append( {
- "measurement": "run_correction",
- "tags": {
- "host": socket.gethostname(),
- "run": self.run,
- "proposal": self.proposal,
- "mem_cells": self.mem_cells,
- "sequence": sequence,
- "module": qm,
- "chunksize": chunksize,
- "detector": self.detector,
- "instrument": self.instrument,
- "filesize": filesize,
- "total_sequences": total_sequences,
- "sequences_module": sequences_per_module,
- },
- "time": datetime.utcnow().isoformat(),
- "fields": {
- "success": success,
- "reason": reason,
- "runtime": runtime,
-
- }
- })
-
- def runtime_summary_entry(self, success, runtime, filesize=None, chunksize=None,
- total_sequences=None, sequences_per_module=None,
- reason=""):
- self.entries.append( {
- "measurement": "run_correction",
- "tags": {
- "host": socket.gethostname(),
- "run": self.run,
- "proposal": self.proposal,
- "mem_cells": self.mem_cells,
- "chunksize": chunksize,
- "detector": self.detector,
- "instrument": self.instrument,
- },
- "time": datetime.utcnow().isoformat(),
- "fields": {
- "success": success,
- "reason": reason,
- "runtime": runtime,
- "filesize": filesize,
- "total_sequences": total_sequences,
- }
- })
-
- def send(self):
- try:
- client = InfluxDBClient('exflqr18318', 8086, 'root', 'root', 'calstats')
- client.write_points(self.entries)
- self.entries = []
- except Exception as e:
- print("Could not send to influx DB: {}".format(e))
-
\ No newline at end of file
diff --git a/docs/source/workflow.rst b/docs/source/workflow.rst
index b41e6852a..8b7d4744a 100644
--- a/docs/source/workflow.rst
+++ b/docs/source/workflow.rst
@@ -258,31 +258,6 @@ Once a stable point is reached, database access can be enabled according to the
documentation.
-Providing Performance Statistics
---------------------------------
-
-The final step in notebook development should be to inject performance parameters into the
-InfluxDB_ installation tracking these. This can be done relatively easy via the interfaces
-provided in the `cal_tools` subpackage::
-
- from cal_tools.tools import get_notebook_name
- from cal_tools.influx import InfluxLogger
- from datetime import datetime
-
- logger = InfluxLogger(detector="LPD", instrument=instrument, mem_cells=mem_cells,
- notebook=get_notebook_name(), proposal=proposal)
-
- start = datetime.now()
-
- # ... do something that takes time
-
- duration = (datetime.now()-start).total_seconds()
- logger.runtime_summary_entry(success=True, runtime=duration,
- total_sequences=total_sequences,
- filesize=total_file_size)
-
-
-
Testing
-------
@@ -323,5 +298,4 @@ documentation.
.. _matplotlib: https://matplotlib.org/
.. _numpy: http://www.numpy.org/
.. _h5py: https://www.h5py.org/
-.. _iCalibrationDB: https://in.xfel.eu/readthedocs/docs/icalibrationdb/en/latest/
-.. _InfluxDB: https://www.influxdata.com/
+.. _iCalibrationDB: https://in.xfel.eu/readthedocs/docs/icalibrationdb/en/latest/
\ No newline at end of file
diff --git a/notebooks/AGIPD/AGIPD_Characterize_Gain_Combine_NBC.ipynb b/notebooks/AGIPD/AGIPD_Characterize_Gain_Combine_NBC.ipynb
index a89861ebf..67525d3db 100644
--- a/notebooks/AGIPD/AGIPD_Characterize_Gain_Combine_NBC.ipynb
+++ b/notebooks/AGIPD/AGIPD_Characterize_Gain_Combine_NBC.ipynb
@@ -125,7 +125,6 @@
"\n",
"from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
"from cal_tools.tools import gain_map_files, parse_runs, run_prop_seq_from_path, get_notebook_name\n",
- "from cal_tools.influx import InfluxLogger\n",
"\n",
"from cal_tools.plotting import show_overview, plot_badpix_3d\n",
"from datetime import datetime\n",
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index c6bf10fa1..43d6d2cd4 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -107,7 +107,6 @@
" map_gain_stages, parse_runs,\n",
" run_prop_seq_from_path, save_const_to_h5,\n",
" send_to_db)\n",
- "from cal_tools.influx import InfluxLogger\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.plotting import (create_constant_overview,\n",
" plot_badpix_3d, show_processed_modules,\n",
@@ -161,10 +160,7 @@
"\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"print(f\"Instrument {instrument}\")\n",
- "print(f\"Detector instance {dinstance}\")\n",
- "\n",
- "logger = InfluxLogger(detector=\"AGIPD\", instrument=instrument, mem_cells=mem_cells,\n",
- " notebook=get_notebook_name(), proposal=prop)"
+ "print(f\"Detector instance {dinstance}\")"
]
},
{
@@ -459,10 +455,7 @@
" \n",
"\n",
"duration = (datetime.now() - start).total_seconds()\n",
- "logger.runtime_summary_entry(success=True, runtime=duration,\n",
- " total_sequences=total_sequences,\n",
- " filesize=total_file_size)\n",
- "logger.send()\n",
+ "\n",
"max_cells = np.max(all_cells)\n",
"print(f\"Using {max_cells} memory cells\")\n",
"acq_rate = np.max(all_acq_rate)\n",
diff --git a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
index 59c499d2a..47098ca1f 100644
--- a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
+++ b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
@@ -90,7 +90,6 @@
"\n",
"from cal_tools.agipdlib import get_acq_rate, get_gain_setting, get_num_cells\n",
"from cal_tools.enums import BadPixels\n",
- "from cal_tools.influx import InfluxLogger\n",
"from cal_tools.plotting import plot_badpix_3d, show_overview\n",
"from cal_tools.tools import (gain_map_files, get_dir_creation_date, get_notebook_name,\n",
" parse_runs, run_prop_seq_from_path, get_dir_creation_date, send_to_db)\n",
@@ -115,8 +114,6 @@
"print(\"Interlaced mode: {}\".format(IL_MODE))\n",
"\n",
"run, prop, seq = run_prop_seq_from_path(in_folder)\n",
- "logger = InfluxLogger(detector=\"AGIPD\", instrument=instrument, mem_cells=mem_cells,\n",
- " notebook=get_notebook_name(), proposal=prop)\n",
"\n",
"loc = None\n",
"if instrument == \"SPB\":\n",
@@ -384,10 +381,7 @@
"p = partial(read_and_merge_module_data, maxcells, path_temp, image_name_temp,\n",
" runs, seqs, IL_MODE, rawversion, instrument)\n",
"# chunk this a bit, so that we don't overuse available memory\n",
- "res = list(map(p, modules))\n",
- "duration = (datetime.now()-start).total_seconds()\n",
- "logger.runtime_summary_entry(success=True, runtime=duration)\n",
- "logger.send()"
+ "res = list(map(p, modules))\n"
]
},
{
@@ -1312,11 +1306,7 @@
" 'ch': ch,\n",
" 'ah': ah,\n",
" 'devh': devh,\n",
- " })\n",
- " \n",
- "duration = (datetime.now()-start).total_seconds()\n",
- "logger.runtime_summary_entry(success=True, runtime=duration)\n",
- "logger.send()"
+ " })"
]
},
{
diff --git a/notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb b/notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb
index f727a1a94..8fc4b3c22 100644
--- a/notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb
+++ b/notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb
@@ -103,7 +103,6 @@
" run_prop_seq_from_path, get_notebook_name, \n",
" get_dir_creation_date, save_const_to_h5,\n",
" get_random_db_interface, send_to_db)\n",
- "from cal_tools.influx import InfluxLogger\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.plotting import show_overview, plot_badpix_3d, create_constant_overview\n",
"\n",
@@ -133,8 +132,6 @@
"\n",
"\n",
"run, prop, seq = run_prop_seq_from_path(in_folder)\n",
- "logger = InfluxLogger(detector=\"AGIPD\", instrument=instrument, mem_cells=mem_cells,\n",
- " notebook=get_notebook_name(), proposal=prop)\n",
"\n",
"print(f\"Using {creation_time} as creation time of constant.\")\n",
"\n",
@@ -370,12 +367,6 @@
" gainstd_g[..., gg] = gains_std\n",
" badpix_g[...,gg] = bp\n",
"\n",
- "duration = (datetime.now()-start).total_seconds()\n",
- "logger.runtime_summary_entry(success=True, runtime=duration,\n",
- " total_sequences=total_sequences,\n",
- " filesize=total_file_size)\n",
- "logger.send()\n",
- "\n",
"max_cells = np.max(all_cells)\n",
"print(f\"Using {max_cells} memory cells\")\n",
"\n",
@@ -869,4 +860,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
-}
+}
\ No newline at end of file
diff --git a/notebooks/AGIPD/playground/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb b/notebooks/AGIPD/playground/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
index 9f4a3351b..26671f710 100644
--- a/notebooks/AGIPD/playground/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
+++ b/notebooks/AGIPD/playground/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
@@ -99,7 +99,6 @@
"from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
"from cal_tools.agipdlib import get_num_cells, get_acq_rate, get_gain_setting\n",
"from cal_tools.enums import BadPixels\n",
- "from cal_tools.influx import InfluxLogger\n",
"from cal_tools.plotting import show_overview, plot_badpix_3d\n",
"from cal_tools.tools import gain_map_files, parse_runs, run_prop_seq_from_path, get_notebook_name, get_dir_creation_date, get_random_db_interface\n"
]
@@ -179,10 +178,7 @@
"fbase = \"{}/{{}}/RAW-{{}}-AGIPD{{:02d}}-S{{:05d}}.h5\".format(in_folder)\n",
"gains = np.arange(3)\n",
"\n",
- "\n",
"run, prop, seq = run_prop_seq_from_path(in_folder)\n",
- "logger = InfluxLogger(detector=\"AGIPD\", instrument=instrument, mem_cells=mem_cells,\n",
- " notebook=get_notebook_name(), proposal=prop)\n",
"\n",
"# extract the memory cells and acquisition rate from \n",
"# the file of the first module, first sequence and first run\n",
@@ -787,10 +783,7 @@
"p = partial(relgain_single_module, fbase, runs, sequences,\n",
" peak_estimates, peak_heights, peak_sigma, memory_cells,\n",
" sensor_size, block_size, IL_MODE, profile, limit_trains_eval, rawversion, instrument)\n",
- "res_gain = list(map(p, inp)) # don't run concurently as inner function are parelllized\n",
- "duration = (datetime.now()-start).total_seconds()\n",
- "logger.runtime_summary_entry(success=True, runtime=duration)\n",
- "logger.send()"
+ "res_gain = list(map(p, inp)) # don't run concurently as inner function are parelllized\n"
]
},
{
@@ -1599,4 +1592,4 @@
},
"nbformat": 4,
"nbformat_minor": 1
-}
+}
\ No newline at end of file
diff --git a/notebooks/AGIPD/playground/Chracterize_AGIPD_Gain_PC_mlearn.ipynb b/notebooks/AGIPD/playground/Chracterize_AGIPD_Gain_PC_mlearn.ipynb
index 238701ecc..4761a412a 100644
--- a/notebooks/AGIPD/playground/Chracterize_AGIPD_Gain_PC_mlearn.ipynb
+++ b/notebooks/AGIPD/playground/Chracterize_AGIPD_Gain_PC_mlearn.ipynb
@@ -106,7 +106,6 @@
"from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
"\n",
"from cal_tools.cal_tools import gain_map_files, parse_runs, run_prop_seq_from_path, get_notebook_name\n",
- "from cal_tools.influx import InfluxLogger\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.plotting import show_overview, plot_badpix_3d\n",
"from datetime import datetime\n",
@@ -124,9 +123,7 @@
"print(\"Sequences: {}\".format(seqs))\n",
"print(\"Interlaced mode: {}\".format(IL_MODE))\n",
"\n",
- "run, prop, seq = run_prop_seq_from_path(in_folder)\n",
- "logger = InfluxLogger(detector=\"AGIPD\", instrument=instrument, mem_cells=mem_cells,\n",
- " notebook=get_notebook_name(), proposal=prop)"
+ "run, prop, seq = run_prop_seq_from_path(in_folder)\n"
]
},
{
@@ -303,10 +300,7 @@
"p = partial(read_and_merge_module_data, maxcells, path_temp, image_name_temp,\n",
" runs, seqs, bursts_per_file, IL_MODE, rawversion)\n",
"# chunk this a bit, so that we don't overuse available memory\n",
- "res = list(map(p, modules))\n",
- "duration = (datetime.now()-start).total_seconds()\n",
- "logger.runtime_summary_entry(success=True, runtime=duration)\n",
- "logger.send()"
+ "res = list(map(p, modules))"
]
},
{
@@ -613,4 +607,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
-}
+}
\ No newline at end of file
diff --git a/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb b/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
index dcbd68882..5b30d81c6 100644
--- a/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
+++ b/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
@@ -87,7 +87,6 @@
"import tabulate\n",
"\n",
"from cal_tools.enums import BadPixels\n",
- "from cal_tools.influx import InfluxLogger\n",
"from cal_tools.plotting import (create_constant_overview,\n",
" plot_badpix_3d, show_overview,\n",
" show_processed_modules)\n",
@@ -124,8 +123,6 @@
" print(f\"Using {creation_time} as creation time of constant.\")\n",
"\n",
"run, prop, seq = run_prop_seq_from_path(in_folder)\n",
- "logger = InfluxLogger(detector=\"DSSC\", instrument=instrument, mem_cells=mem_cells,\n",
- " notebook=get_notebook_name(), proposal=prop)\n",
"\n",
"dinstance = \"DSSC1M1\"\n",
"\n",
@@ -345,11 +342,6 @@
" badpix_g[qm][...] = bp\n",
" gg +=1\n",
"\n",
- "duration = (datetime.now()-start).total_seconds()\n",
- "logger.runtime_summary_entry(success=True, runtime=duration,\n",
- " total_sequences=total_sequences,\n",
- " filesize=total_file_size)\n",
- "logger.send()\n",
"if len(all_cells) > 0:\n",
" max_cells = np.max(all_cells)\n",
" print(f\"Using {max_cells} memory cells\")\n",
diff --git a/notebooks/LPD/Characterize_LPD_GAIN_CI_per_pixel_NBC.ipynb b/notebooks/LPD/Characterize_LPD_GAIN_CI_per_pixel_NBC.ipynb
index ce8c1d522..b8b17cf51 100644
--- a/notebooks/LPD/Characterize_LPD_GAIN_CI_per_pixel_NBC.ipynb
+++ b/notebooks/LPD/Characterize_LPD_GAIN_CI_per_pixel_NBC.ipynb
@@ -79,7 +79,6 @@
"\n",
"from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
"from cal_tools.tools import gain_map_files, parse_runs, run_prop_seq_from_path, get_notebook_name, get_dir_creation_date\n",
- "from cal_tools.influx import InfluxLogger\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.plotting import show_overview, plot_badpix_3d, create_constant_overview\n",
"\n",
@@ -111,11 +110,6 @@
"fbase = \"{}/data_q{{}}m{{}}_{{}}.h5\".format(in_folder)\n",
"h5path = \"/data\"\n",
"\n",
- "\n",
- "\n",
- "logger = InfluxLogger(detector=\"LPD\", instrument=instrument, mem_cells=mem_cells,\n",
- " notebook=get_notebook_name())\n",
- "\n",
"if offset_store != \"\":\n",
" db_input = False\n",
" \n",
diff --git a/notebooks/LPD/LPDChar_Darks_NBC.ipynb b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
index 5d042bbe4..c5b743631 100644
--- a/notebooks/LPD/LPDChar_Darks_NBC.ipynb
+++ b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
@@ -86,7 +86,6 @@
"import tabulate\n",
"\n",
"from cal_tools.enums import BadPixels\n",
- "from cal_tools.influx import InfluxLogger\n",
"from cal_tools.plotting import (show_overview, plot_badpix_3d, \n",
" create_constant_overview,\n",
" show_processed_modules)\n",
@@ -144,8 +143,6 @@
" print(\"Using {} as creation time\".format(creation_time))\n",
"\n",
"run, prop, seq = run_prop_seq_from_path(in_folder)\n",
- "logger = InfluxLogger(detector=\"LPD\", instrument=instrument, mem_cells=mem_cells,\n",
- " notebook=get_notebook_name(), proposal=prop)\n",
"\n",
"cal_db_interface = get_random_db_interface(cal_db_interface)\n",
"\n",
@@ -327,13 +324,7 @@
"\n",
" hn, cn = np.histogram(data, bins=20)\n",
" print(f\"{gain_names[gg]} gain, Capacitor {cap}, Module: {qm}. \"\n",
- " f\"Number of processed trains per cell: {data.shape[0]}.\")\n",
- "\n",
- "duration = (datetime.now()-start).total_seconds()\n",
- "logger.runtime_summary_entry(success=True, runtime=duration,\n",
- " total_sequences=total_sequences,\n",
- " filesize=total_file_size)\n",
- "logger.send()"
+ " f\"Number of processed trains per cell: {data.shape[0]}.\")"
]
},
{
diff --git a/notebooks/LPD/LPD_Correct_and_Verify.ipynb b/notebooks/LPD/LPD_Correct_and_Verify.ipynb
index 776c2d6fa..d2599fb38 100644
--- a/notebooks/LPD/LPD_Correct_and_Verify.ipynb
+++ b/notebooks/LPD/LPD_Correct_and_Verify.ipynb
@@ -118,14 +118,12 @@
"from cal_tools.tools import (gain_map_files, parse_runs, run_prop_seq_from_path, \n",
" get_notebook_name, get_dir_creation_date, \n",
" get_constant_from_db, map_modules_from_folder)\n",
- "from cal_tools.influx import InfluxLogger\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.plotting import show_overview, plot_badpix_3d, create_constant_overview\n",
"from cal_tools.lpdlib import LpdCorrections\n",
"from datetime import datetime\n",
"\n",
"from collections import OrderedDict\n",
- "from influxdb import InfluxDBClient\n",
"from datetime import datetime\n",
"\n",
"print(\"Connecting to profile {}\".format(cluster_profile))\n",
@@ -152,10 +150,6 @@
"\n",
"_, proposal, seq = run_prop_seq_from_path(in_folder)\n",
"instrument = karabo_id.split(\"_\")[0]\n",
- "logger = InfluxLogger(detector=\"LPD\", instrument=instrument, mem_cells=mem_cells,\n",
- " notebook=get_notebook_name(), proposal=proposal)\n",
- "\n",
- "client = InfluxDBClient('exflqr18318', 8086, 'root', 'root', 'calstats')\n",
"\n",
"mark_non_lin_region = not dont_mark_non_lin_region\n",
"linear_between = [linear_between_high_gain, linear_between_med_gain, linear_between_low_gain]\n",
@@ -245,39 +239,9 @@
" from datetime import datetime\n",
" import re\n",
" import os\n",
- " from influxdb import InfluxDBClient\n",
" from cal_tools.enums import BadPixels\n",
" from cal_tools.lpdlib import LpdCorrections\n",
- " \n",
- " client = InfluxDBClient('exflqr18318', 8086, 'root', 'root', 'calstats')\n",
- "\n",
- " def create_influx_entry(run, proposal, qm, sequence, filesize, chunksize,\n",
- " total_sequences, success, runtime, reason=\"\"):\n",
- " return {\n",
- " \"measurement\": \"run_correction\",\n",
- " \"tags\": {\n",
- " \"host\": socket.gethostname(),\n",
- " \"run\": run,\n",
- " \"proposal\": proposal,\n",
- " \"mem_cells\": max_cells,\n",
- " \"sequence\": sequence,\n",
- " \"module\": qm,\n",
- " \"filesize\": filesize,\n",
- " \"chunksize\": chunksize,\n",
- " \"total_sequences\": total_sequences,\n",
- " \"sequences_module\": sequences_qm[qm],\n",
- " \"detector\": \"LPD\",\n",
- " \"instrument\": \"FXE\",\n",
- " \n",
- " },\n",
- " \"time\": datetime.utcnow().isoformat(),\n",
- " \"fields\": {\n",
- " \"success\": success,\n",
- " \"reason\": reason,\n",
- " \"runtime\": runtime, \n",
- " }\n",
- " }\n",
- " \n",
+ "\n",
" hists_signal_low = None\n",
" hists_signal_high = None\n",
" hists_gain_vs_signal = None\n",
@@ -343,14 +307,6 @@
" reason = \"Error\"\n",
" err = e\n",
"\n",
- " finally:\n",
- " run = re.findall(r'.*r([0-9]{4}).*', filename)[0]\n",
- " proposal = re.findall(r'.*p([0-9]{6}).*', filename)[0]\n",
- " sequence = re.findall(r'.*S([0-9]{5}).*', filename)[0]\n",
- " filesize = os.path.getsize(filename)\n",
- " duration = (datetime.now()-start).total_seconds()\n",
- " influx = create_influx_entry(run, proposal, qm, sequence, filesize, CHUNK_SIZE, total_sequences, success, duration, reason)\n",
- " #client.write_points([influx])\n",
" return hists_signal_low, hists_signal_high, hists_gain_vs_signal, low_edges, high_edges, signal_edges, when, qm, err\n",
" \n",
"done = False\n",
diff --git a/notebooks/LPD/LPD_FlatField_Radial_per_pixel_CI_NBC.ipynb b/notebooks/LPD/LPD_FlatField_Radial_per_pixel_CI_NBC.ipynb
index d0781a76d..5eeebf141 100644
--- a/notebooks/LPD/LPD_FlatField_Radial_per_pixel_CI_NBC.ipynb
+++ b/notebooks/LPD/LPD_FlatField_Radial_per_pixel_CI_NBC.ipynb
@@ -72,13 +72,11 @@
"from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
"from cal_tools.tools import (gain_map_files, parse_runs, run_prop_seq_from_path, get_notebook_name,\n",
" get_dir_creation_date, get_constant_from_db)\n",
- "from cal_tools.influx import InfluxLogger\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.plotting import show_overview, plot_badpix_3d, create_constant_overview\n",
"import cal_tools.metrology as metro\n",
"\n",
"from collections import OrderedDict\n",
- "from influxdb import InfluxDBClient\n",
"\n",
"creation_time = None\n",
"if use_dir_creation_date:\n",
@@ -104,10 +102,6 @@
" os.makedirs(out_folder)\n",
"\n",
"run, proposal, seq = run_prop_seq_from_path(in_folder)\n",
- "logger = InfluxLogger(detector=\"LPD\", instrument=instrument, mem_cells=mem_cells,\n",
- " notebook=get_notebook_name(), proposal=proposal)\n",
- "\n",
- "client = InfluxDBClient('exflqr18318', 8086, 'root', 'root', 'calstats')\n",
"\n",
"# start lower right and then counter-clock-wise\n",
"dc = beam_center_offset\n",
diff --git a/requirements.txt b/requirements.txt
index 34e2726c2..648efef27 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,7 +11,6 @@ fabio == 0.9.0
gitpython == 3.1.0
h5py == 2.10.0
iminuit == 1.3.8
-influxdb == 5.2.3
ipyparallel == 6.2.4
ipykernel == 5.1.4
ipython == 7.12.0
--
GitLab