Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
P
pycalibration
Manage
Activity
Members
Labels
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Deploy
Model registry
Analyze
Contributor analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
calibration
pycalibration
Commits
ec2301a2
Commit
ec2301a2
authored
4 years ago
by
David Hammer
Browse files
Options
Downloads
Patches
Plain Diff
Cleanup
parent
4375aa50
No related branches found
Branches containing commit
No related tags found
Tags containing commit
2 merge requests
!451
Feat/agipd fixed gain correction
,
!438
Feat/agipd add fixed gain mode to dark notebook
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+12
-27
12 additions, 27 deletions
notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
with
12 additions
and
27 deletions
notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+
12
−
27
View file @
ec2301a2
...
@@ -51,7 +51,7 @@
...
@@ -51,7 +51,7 @@
"xray_gain = True # do relative gain correction based on xray data\n",
"xray_gain = True # do relative gain correction based on xray data\n",
"blc_noise = False # if set, baseline correction via noise peak location is attempted\n",
"blc_noise = False # if set, baseline correction via noise peak location is attempted\n",
"blc_stripes = False # if set, baseline corrected via stripes\n",
"blc_stripes = False # if set, baseline corrected via stripes\n",
"blc_hmatch = False # if set, base line correction via histogram matching is attempted
\n",
"blc_hmatch = False # if set, base line correction via histogram matching is attempted\n",
"match_asics = False # if set, inner ASIC borders are matched to the same signal level\n",
"match_asics = False # if set, inner ASIC borders are matched to the same signal level\n",
"adjust_mg_baseline = False # adjust medium gain baseline to match highest high gain value"
"adjust_mg_baseline = False # adjust medium gain baseline to match highest high gain value"
]
]
...
@@ -206,14 +206,11 @@
...
@@ -206,14 +206,11 @@
"outputs": [],
"outputs": [],
"source": [
"source": [
"def retrieve_constants(\n",
"def retrieve_constants(\n",
" qm_files: List[Path],\n",
" qm_files: List[Path], qm: str, karabo_da: str, idx: int\n",
" qm: str,\n",
" karabo_da: str,\n",
" idx: int\n",
") -> Tuple[str, str, float, float, str, dict]:\n",
") -> Tuple[str, str, float, float, str, dict]:\n",
" \"\"\"\n",
" \"\"\"\n",
" Retrieve constants for a module.\n",
" Retrieve constants for a module.\n",
"
\n",
"\n",
" :return:\n",
" :return:\n",
" qm: module virtual name i.e. Q1M1.\n",
" qm: module virtual name i.e. Q1M1.\n",
" karabo_da: karabo data aggregator.\n",
" karabo_da: karabo data aggregator.\n",
...
@@ -237,7 +234,7 @@
...
@@ -237,7 +234,7 @@
" if local_max_cells is None:\n",
" if local_max_cells is None:\n",
" raise ValueError(f\"No raw images found for {qm} for all sequences\")\n",
" raise ValueError(f\"No raw images found for {qm} for all sequences\")\n",
"\n",
"\n",
" if acq_rate == 0:
\n",
" if acq_rate == 0:\n",
" local_acq_rate = agipdlib.get_acq_rate(fast_paths=(f, karabo_id, idx))\n",
" local_acq_rate = agipdlib.get_acq_rate(fast_paths=(f, karabo_id, idx))\n",
" else:\n",
" else:\n",
" local_acq_rate = acq_rate\n",
" local_acq_rate = acq_rate\n",
...
@@ -245,7 +242,7 @@
...
@@ -245,7 +242,7 @@
" # avoid retrieving constant, if requested.\n",
" # avoid retrieving constant, if requested.\n",
" if nodb_with_dark:\n",
" if nodb_with_dark:\n",
" return\n",
" return\n",
"
\n",
"\n",
" const_dict = agipdlib.assemble_constant_dict(\n",
" const_dict = agipdlib.assemble_constant_dict(\n",
" corr_bools,\n",
" corr_bools,\n",
" pc_bools,\n",
" pc_bools,\n",
...
@@ -360,13 +357,11 @@
...
@@ -360,13 +357,11 @@
"source": [
"source": [
"mod_dev = dict()\n",
"mod_dev = dict()\n",
"mdata_dict = dict()\n",
"mdata_dict = dict()\n",
"for r in results:\n",
"for qm, md_dict, karabo_da, acq_rate, max_cells, err in results:\n",
" if r:\n",
" mod_dev[karabo_da] = {\"mod\": qm, \"err\": err}\n",
" qm, md_dict, karabo_da, acq_rate, max_cells, err = r\n",
" if err:\n",
" mod_dev[karabo_da] = {\"mod\": qm, \"err\": err}\n",
" print(f\"Error for module {qm}: {err}\")\n",
" if err:\n",
" mdata_dict[karabo_da] = md_dict\n",
" print(f\"Error for module {qm}: {err}\")\n",
" mdata_dict[karabo_da] = md_dict\n",
"# check if it is requested not to retrieve any constants from the database\n",
"# check if it is requested not to retrieve any constants from the database\n",
"if nodb_with_dark:\n",
"if nodb_with_dark:\n",
" print(\"No constants were retrieved as calibrated files will be used.\")\n",
" print(\"No constants were retrieved as calibrated files will be used.\")\n",
...
@@ -391,12 +386,11 @@
...
@@ -391,12 +386,11 @@
"metadata": {},
"metadata": {},
"outputs": [],
"outputs": [],
"source": [
"source": [
"print(\"Constants are retrieved with creation time: \")\n",
"print(\"Constants are retrieved with creation time:\")\n",
"i = 0\n",
"timestamps = {}\n",
"timestamps = {}\n",
"\n",
"\n",
"for k_da, dinfo in mod_dev.items():\n",
"for k_da, dinfo in mod_dev.items():\n",
" print(dinfo[
\"
mod
\"], \"
:\")\n",
" print(
f\"{
dinfo[
'
mod
']}
:\")\n",
" module_timestamps = {}\n",
" module_timestamps = {}\n",
" module_name = dinfo[\"mod\"]\n",
" module_name = dinfo[\"mod\"]\n",
" if k_da in mdata_dict:\n",
" if k_da in mdata_dict:\n",
...
@@ -404,8 +398,6 @@
...
@@ -404,8 +398,6 @@
" if hasattr(mdata[\"creation-time\"], 'strftime'):\n",
" if hasattr(mdata[\"creation-time\"], 'strftime'):\n",
" mdata[\"creation-time\"] = mdata[\"creation-time\"].strftime('%y-%m-%d %H:%M')\n",
" mdata[\"creation-time\"] = mdata[\"creation-time\"].strftime('%y-%m-%d %H:%M')\n",
" print(f'{cname:.<12s}', mdata[\"creation-time\"])\n",
" print(f'{cname:.<12s}', mdata[\"creation-time\"])\n",
" # Store few time stamps if exists\n",
" # Add NA to keep array structure\n",
" for cname in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
" for cname in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
" if k_da not in mdata_dict or dinfo[\"err\"]:\n",
" if k_da not in mdata_dict or dinfo[\"err\"]:\n",
" module_timestamps[cname] = \"Err\"\n",
" module_timestamps[cname] = \"Err\"\n",
...
@@ -419,13 +411,6 @@
...
@@ -419,13 +411,6 @@
" module_timestamps[cname] = \"NA\"\n",
" module_timestamps[cname] = \"NA\"\n",
" timestamps[module_name] = module_timestamps\n",
" timestamps[module_name] = module_timestamps\n",
"\n",
"\n",
" i += 1\n",
" if sequences:\n",
" seq_num = sequences[0]\n",
" else:\n",
" # if sequences[0] changed to None as it was -1\n",
" seq_num = 0\n",
"\n",
"time_summary = metadata.setdefault(\"retrieved-constants\", {}).setdefault(\"time-summary\", {})\n",
"time_summary = metadata.setdefault(\"retrieved-constants\", {}).setdefault(\"time-summary\", {})\n",
"time_summary[\"SAll\"] = timestamps\n",
"time_summary[\"SAll\"] = timestamps\n",
"\n",
"\n",
...
...
%% Cell type:markdown id: tags:
%% Cell type:markdown id: tags:
# AGIPD Retrieving Constants Pre-correction #
# AGIPD Retrieving Constants Pre-correction #
Author: European XFEL Detector Group, Version: 1.0
Author: European XFEL Detector Group, Version: 1.0
Retrieving Required Constants for Offline Calibration of the AGIPD Detector
Retrieving Required Constants for Offline Calibration of the AGIPD Detector
%% Cell type:code id: tags:
%% Cell type:code id: tags:
```
python
```
python
in_folder
=
"
/gpfs/exfel/exp/SPB/202030/p900119/raw
"
# the folder to read data from, required
in_folder
=
"
/gpfs/exfel/exp/SPB/202030/p900119/raw
"
# the folder to read data from, required
out_folder
=
"
/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_
"
# the folder to output to, required
out_folder
=
"
/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_
"
# the folder to output to, required
sequences
=
[
-
1
]
# sequences to correct, set to -1 for all, range allowed
sequences
=
[
-
1
]
# sequences to correct, set to -1 for all, range allowed
modules
=
[
-
1
]
# modules to correct, set to -1 for all, range allowed
modules
=
[
-
1
]
# modules to correct, set to -1 for all, range allowed
run
=
80
# runs to process, required
run
=
80
# runs to process, required
karabo_id
=
"
SPB_DET_AGIPD1M-1
"
# karabo karabo_id
karabo_id
=
"
SPB_DET_AGIPD1M-1
"
# karabo karabo_id
karabo_da
=
[
'
-1
'
]
# a list of data aggregators names, Default [-1] for selecting all data aggregators
karabo_da
=
[
'
-1
'
]
# a list of data aggregators names, Default [-1] for selecting all data aggregators
path_template
=
'
RAW-R{:04d}-{}-S{:05d}.h5
'
# the template to use to access data
path_template
=
'
RAW-R{:04d}-{}-S{:05d}.h5
'
# the template to use to access data
h5path_ctrl
=
'
/CONTROL/{}/MDL/FPGA_COMP_TEST
'
# path to control information
h5path_ctrl
=
'
/CONTROL/{}/MDL/FPGA_COMP_TEST
'
# path to control information
karabo_id_control
=
"
SPB_IRU_AGIPD1M1
"
# karabo-id for control device
karabo_id_control
=
"
SPB_IRU_AGIPD1M1
"
# karabo-id for control device
karabo_da_control
=
'
AGIPD1MCTRL00
'
# karabo DA for control infromation
karabo_da_control
=
'
AGIPD1MCTRL00
'
# karabo DA for control infromation
use_dir_creation_date
=
True
# use the creation data of the input dir for database queries
use_dir_creation_date
=
True
# use the creation data of the input dir for database queries
cal_db_interface
=
"
tcp://max-exfl016:8015#8045
"
# the database interface to use
cal_db_interface
=
"
tcp://max-exfl016:8015#8045
"
# the database interface to use
creation_date_offset
=
"
00:00:00
"
# add an offset to creation date, e.g. to get different constants
creation_date_offset
=
"
00:00:00
"
# add an offset to creation date, e.g. to get different constants
slopes_ff_from_files
=
""
# Path to locally stored SlopesFF and BadPixelsFF constants
slopes_ff_from_files
=
""
# Path to locally stored SlopesFF and BadPixelsFF constants
calfile
=
""
# path to calibration file. Leave empty if all data should come from DB
calfile
=
""
# path to calibration file. Leave empty if all data should come from DB
nodb
=
False
# if set only file-based constants will be used
nodb
=
False
# if set only file-based constants will be used
mem_cells
=
0
# number of memory cells used, set to 0 to automatically infer
mem_cells
=
0
# number of memory cells used, set to 0 to automatically infer
bias_voltage
=
300
bias_voltage
=
300
acq_rate
=
0.
# the detector acquisition rate, use 0 to try to auto-determine
acq_rate
=
0.
# the detector acquisition rate, use 0 to try to auto-determine
gain_setting
=
0.1
# the gain setting, use 0.1 to try to auto-determine
gain_setting
=
0.1
# the gain setting, use 0.1 to try to auto-determine
photon_energy
=
9.2
# photon energy in keV
photon_energy
=
9.2
# photon energy in keV
max_cells_db_dark
=
0
# set to a value different than 0 to use this value for dark data DB queries
max_cells_db_dark
=
0
# set to a value different than 0 to use this value for dark data DB queries
max_cells_db
=
0
# set to a value different than 0 to use this value for DB queries
max_cells_db
=
0
# set to a value different than 0 to use this value for DB queries
# Correction Booleans
# Correction Booleans
only_offset
=
False
# Apply only Offset correction. if False, Offset is applied by Default. if True, Offset is only applied.
only_offset
=
False
# Apply only Offset correction. if False, Offset is applied by Default. if True, Offset is only applied.
rel_gain
=
False
# do relative gain correction based on PC data
rel_gain
=
False
# do relative gain correction based on PC data
xray_gain
=
True
# do relative gain correction based on xray data
xray_gain
=
True
# do relative gain correction based on xray data
blc_noise
=
False
# if set, baseline correction via noise peak location is attempted
blc_noise
=
False
# if set, baseline correction via noise peak location is attempted
blc_stripes
=
False
# if set, baseline corrected via stripes
blc_stripes
=
False
# if set, baseline corrected via stripes
blc_hmatch
=
False
# if set, base line correction via histogram matching is attempted
blc_hmatch
=
False
# if set, base line correction via histogram matching is attempted
match_asics
=
False
# if set, inner ASIC borders are matched to the same signal level
match_asics
=
False
# if set, inner ASIC borders are matched to the same signal level
adjust_mg_baseline
=
False
# adjust medium gain baseline to match highest high gain value
adjust_mg_baseline
=
False
# adjust medium gain baseline to match highest high gain value
```
```
%% Cell type:code id: tags:
%% Cell type:code id: tags:
```
python
```
python
# Fill dictionaries comprising bools and arguments for correction and data analysis
# Fill dictionaries comprising bools and arguments for correction and data analysis
# Here the hierarichy and dependencies for correction booleans are defined
# Here the hierarichy and dependencies for correction booleans are defined
corr_bools
=
{}
corr_bools
=
{}
# offset is at the bottom of AGIPD correction pyramid.
# offset is at the bottom of AGIPD correction pyramid.
corr_bools
[
"
only_offset
"
]
=
only_offset
corr_bools
[
"
only_offset
"
]
=
only_offset
# Dont apply any corrections if only_offset is requested
# Dont apply any corrections if only_offset is requested
if
not
only_offset
:
if
not
only_offset
:
corr_bools
[
"
adjust_mg_baseline
"
]
=
adjust_mg_baseline
corr_bools
[
"
adjust_mg_baseline
"
]
=
adjust_mg_baseline
corr_bools
[
"
rel_gain
"
]
=
rel_gain
corr_bools
[
"
rel_gain
"
]
=
rel_gain
corr_bools
[
"
xray_corr
"
]
=
xray_gain
corr_bools
[
"
xray_corr
"
]
=
xray_gain
corr_bools
[
"
blc_noise
"
]
=
blc_noise
corr_bools
[
"
blc_noise
"
]
=
blc_noise
corr_bools
[
"
blc_hmatch
"
]
=
blc_hmatch
corr_bools
[
"
blc_hmatch
"
]
=
blc_hmatch
```
```
%% Cell type:code id: tags:
%% Cell type:code id: tags:
```
python
```
python
from
typing
import
List
,
Tuple
from
typing
import
List
,
Tuple
import
matplotlib
import
matplotlib
import
numpy
as
np
import
numpy
as
np
matplotlib
.
use
(
"
agg
"
)
matplotlib
.
use
(
"
agg
"
)
import
multiprocessing
import
multiprocessing
from
datetime
import
timedelta
from
datetime
import
timedelta
from
pathlib
import
Path
from
pathlib
import
Path
import
matplotlib.pyplot
as
plt
import
matplotlib.pyplot
as
plt
from
cal_tools
import
agipdlib
,
tools
from
cal_tools
import
agipdlib
,
tools
from
dateutil
import
parser
from
dateutil
import
parser
from
iCalibrationDB
import
Conditions
,
Constants
,
Detectors
from
iCalibrationDB
import
Conditions
,
Constants
,
Detectors
```
```
%% Cell type:code id: tags:
%% Cell type:code id: tags:
```
python
```
python
# slopes_ff_from_files left as str for now
# slopes_ff_from_files left as str for now
in_folder
=
Path
(
in_folder
)
in_folder
=
Path
(
in_folder
)
out_folder
=
Path
(
out_folder
)
out_folder
=
Path
(
out_folder
)
metadata
=
tools
.
CalibrationMetadata
(
out_folder
)
metadata
=
tools
.
CalibrationMetadata
(
out_folder
)
```
```
%% Cell type:code id: tags:
%% Cell type:code id: tags:
```
python
```
python
max_cells
=
mem_cells
max_cells
=
mem_cells
creation_time
=
None
creation_time
=
None
if
use_dir_creation_date
:
if
use_dir_creation_date
:
creation_time
=
tools
.
get_dir_creation_date
(
str
(
in_folder
),
run
)
creation_time
=
tools
.
get_dir_creation_date
(
str
(
in_folder
),
run
)
offset
=
parser
.
parse
(
creation_date_offset
)
offset
=
parser
.
parse
(
creation_date_offset
)
delta
=
timedelta
(
hours
=
offset
.
hour
,
minutes
=
offset
.
minute
,
seconds
=
offset
.
second
)
delta
=
timedelta
(
hours
=
offset
.
hour
,
minutes
=
offset
.
minute
,
seconds
=
offset
.
second
)
creation_time
+=
delta
creation_time
+=
delta
print
(
f
"
Using
{
creation_time
}
as creation time
"
)
print
(
f
"
Using
{
creation_time
}
as creation time
"
)
if
sequences
[
0
]
==
-
1
:
if
sequences
[
0
]
==
-
1
:
sequences
=
None
sequences
=
None
print
(
f
"
Outputting to
{
out_folder
}
"
)
print
(
f
"
Outputting to
{
out_folder
}
"
)
out_folder
.
mkdir
(
parents
=
True
,
exist_ok
=
True
)
out_folder
.
mkdir
(
parents
=
True
,
exist_ok
=
True
)
melt_snow
=
False
if
corr_bools
[
"
only_offset
"
]
else
agipdlib
.
SnowResolution
.
NONE
melt_snow
=
False
if
corr_bools
[
"
only_offset
"
]
else
agipdlib
.
SnowResolution
.
NONE
```
```
%% Cell type:code id: tags:
%% Cell type:code id: tags:
```
python
```
python
control_fn
=
in_folder
/
f
'
r
{
run
:
04
d
}
'
/
f
'
RAW-R
{
run
:
04
d
}
-
{
karabo_da_control
}
-S00000.h5
'
control_fn
=
in_folder
/
f
'
r
{
run
:
04
d
}
'
/
f
'
RAW-R
{
run
:
04
d
}
-
{
karabo_da_control
}
-S00000.h5
'
h5path_ctrl
=
h5path_ctrl
.
format
(
karabo_id_control
)
h5path_ctrl
=
h5path_ctrl
.
format
(
karabo_id_control
)
if
gain_setting
==
0.1
:
if
gain_setting
==
0.1
:
if
creation_time
.
replace
(
tzinfo
=
None
)
<
parser
.
parse
(
'
2020-01-31
'
):
if
creation_time
.
replace
(
tzinfo
=
None
)
<
parser
.
parse
(
'
2020-01-31
'
):
print
(
"
Set gain-setting to None for runs taken before 2020-01-31
"
)
print
(
"
Set gain-setting to None for runs taken before 2020-01-31
"
)
gain_setting
=
None
gain_setting
=
None
else
:
else
:
try
:
try
:
gain_setting
=
agipdlib
.
get_gain_setting
(
str
(
control_fn
),
h5path_ctrl
)
gain_setting
=
agipdlib
.
get_gain_setting
(
str
(
control_fn
),
h5path_ctrl
)
except
Exception
as
e
:
except
Exception
as
e
:
print
(
f
'
ERROR: while reading gain setting from:
\n
{
control_fn
}
'
)
print
(
f
'
ERROR: while reading gain setting from:
\n
{
control_fn
}
'
)
print
(
e
)
print
(
e
)
print
(
"
Set gain setting to 0
"
)
print
(
"
Set gain setting to 0
"
)
gain_setting
=
0
gain_setting
=
0
# Evaluate gain mode (operation mode)
# Evaluate gain mode (operation mode)
gain_mode
=
agipdlib
.
get_gain_mode
(
control_fn
,
h5path_ctrl
)
gain_mode
=
agipdlib
.
get_gain_mode
(
control_fn
,
h5path_ctrl
)
print
(
f
"
Gain setting:
{
gain_setting
}
"
)
print
(
f
"
Gain setting:
{
gain_setting
}
"
)
print
(
f
"
Gain mode:
{
gain_mode
.
name
}
"
)
print
(
f
"
Gain mode:
{
gain_mode
.
name
}
"
)
print
(
f
"
Detector in use is
{
karabo_id
}
"
)
print
(
f
"
Detector in use is
{
karabo_id
}
"
)
# Extracting Instrument string
# Extracting Instrument string
instrument
=
karabo_id
.
split
(
"
_
"
)[
0
]
instrument
=
karabo_id
.
split
(
"
_
"
)[
0
]
# Evaluate detector instance for mapping
# Evaluate detector instance for mapping
if
instrument
==
"
SPB
"
:
if
instrument
==
"
SPB
"
:
dinstance
=
"
AGIPD1M1
"
dinstance
=
"
AGIPD1M1
"
nmods
=
16
nmods
=
16
elif
instrument
==
"
MID
"
:
elif
instrument
==
"
MID
"
:
dinstance
=
"
AGIPD1M2
"
dinstance
=
"
AGIPD1M2
"
nmods
=
16
nmods
=
16
elif
instrument
==
"
HED
"
:
elif
instrument
==
"
HED
"
:
dinstance
=
"
AGIPD500K
"
dinstance
=
"
AGIPD500K
"
nmods
=
8
nmods
=
8
print
(
f
"
Instrument
{
instrument
}
"
)
print
(
f
"
Instrument
{
instrument
}
"
)
print
(
f
"
Detector instance
{
dinstance
}
"
)
print
(
f
"
Detector instance
{
dinstance
}
"
)
if
karabo_da
[
0
]
==
'
-1
'
:
if
karabo_da
[
0
]
==
'
-1
'
:
if
modules
[
0
]
==
-
1
:
if
modules
[
0
]
==
-
1
:
modules
=
list
(
range
(
nmods
))
modules
=
list
(
range
(
nmods
))
karabo_da
=
[
"
AGIPD{:02d}
"
.
format
(
i
)
for
i
in
modules
]
karabo_da
=
[
"
AGIPD{:02d}
"
.
format
(
i
)
for
i
in
modules
]
else
:
else
:
modules
=
[
int
(
x
[
-
2
:])
for
x
in
karabo_da
]
modules
=
[
int
(
x
[
-
2
:])
for
x
in
karabo_da
]
```
```
%% Cell type:markdown id: tags:
%% Cell type:markdown id: tags:
## Retrieve Constants ##
## Retrieve Constants ##
%% Cell type:code id: tags:
%% Cell type:code id: tags:
```
python
```
python
def
retrieve_constants
(
def
retrieve_constants
(
qm_files
:
List
[
Path
],
qm_files
:
List
[
Path
],
qm
:
str
,
karabo_da
:
str
,
idx
:
int
qm
:
str
,
karabo_da
:
str
,
idx
:
int
)
->
Tuple
[
str
,
str
,
float
,
float
,
str
,
dict
]:
)
->
Tuple
[
str
,
str
,
float
,
float
,
str
,
dict
]:
"""
"""
Retrieve constants for a module.
Retrieve constants for a module.
:return:
:return:
qm: module virtual name i.e. Q1M1.
qm: module virtual name i.e. Q1M1.
karabo_da: karabo data aggregator.
karabo_da: karabo data aggregator.
acq_rate: acquisition rate parameter.
acq_rate: acquisition rate parameter.
max_cells: number of memory cells.
max_cells: number of memory cells.
err: string of faced errors.
err: string of faced errors.
mdata_dict: (DICT) dictionary with the metadata for the retrieved constants.
mdata_dict: (DICT) dictionary with the metadata for the retrieved constants.
"""
"""
err
=
None
err
=
None
if
max_cells
!=
0
:
if
max_cells
!=
0
:
# either use overriding notebook parameter
# either use overriding notebook parameter
local_max_cells
=
max_cells
local_max_cells
=
max_cells
else
:
else
:
# or look around in sequence files
# or look around in sequence files
for
f
in
qm_files
:
for
f
in
qm_files
:
local_max_cells
=
agipdlib
.
get_num_cells
(
f
,
karabo_id
,
idx
)
local_max_cells
=
agipdlib
.
get_num_cells
(
f
,
karabo_id
,
idx
)
if
local_max_cells
is
not
None
:
if
local_max_cells
is
not
None
:
break
break
# maybe we never found this in a sequence file...
# maybe we never found this in a sequence file...
if
local_max_cells
is
None
:
if
local_max_cells
is
None
:
raise
ValueError
(
f
"
No raw images found for
{
qm
}
for all sequences
"
)
raise
ValueError
(
f
"
No raw images found for
{
qm
}
for all sequences
"
)
if
acq_rate
==
0
:
if
acq_rate
==
0
:
local_acq_rate
=
agipdlib
.
get_acq_rate
(
fast_paths
=
(
f
,
karabo_id
,
idx
))
local_acq_rate
=
agipdlib
.
get_acq_rate
(
fast_paths
=
(
f
,
karabo_id
,
idx
))
else
:
else
:
local_acq_rate
=
acq_rate
local_acq_rate
=
acq_rate
# avoid retrieving constant, if requested.
# avoid retrieving constant, if requested.
if
nodb_with_dark
:
if
nodb_with_dark
:
return
return
const_dict
=
agipdlib
.
assemble_constant_dict
(
const_dict
=
agipdlib
.
assemble_constant_dict
(
corr_bools
,
corr_bools
,
pc_bools
,
pc_bools
,
local_max_cells
,
local_max_cells
,
bias_voltage
,
bias_voltage
,
gain_setting
,
gain_setting
,
local_acq_rate
,
local_acq_rate
,
photon_energy
,
photon_energy
,
gain_mode
=
gain_mode
,
gain_mode
=
gain_mode
,
beam_energy
=
None
,
beam_energy
=
None
,
only_dark
=
only_dark
,
only_dark
=
only_dark
,
)
)
# Retrieve multiple constants through an input dictionary
# Retrieve multiple constants through an input dictionary
# to return a dict of useful metadata.
# to return a dict of useful metadata.
mdata_dict
=
dict
()
mdata_dict
=
dict
()
mdata_dict
[
'
constants
'
]
=
dict
()
mdata_dict
[
'
constants
'
]
=
dict
()
mdata_dict
[
'
physical-detector-unit
'
]
=
None
# initialization
mdata_dict
[
'
physical-detector-unit
'
]
=
None
# initialization
for
cname
,
cval
in
const_dict
.
items
():
for
cname
,
cval
in
const_dict
.
items
():
print
(
cname
)
print
(
cname
)
print
(
cval
)
print
(
cval
)
# saving metadata in a dict
# saving metadata in a dict
mdata_dict
[
'
constants
'
][
cname
]
=
dict
()
mdata_dict
[
'
constants
'
][
cname
]
=
dict
()
if
slopes_ff_from_files
and
cname
in
[
"
SlopesFF
"
,
"
BadPixelsFF
"
]:
if
slopes_ff_from_files
and
cname
in
[
"
SlopesFF
"
,
"
BadPixelsFF
"
]:
mdata_dict
[
'
constants
'
][
cname
][
"
file-path
"
]
=
f
"
{
slopes_ff_from_files
}
/slopesff_bpmask_module_
{
qm
}
.h5
"
mdata_dict
[
'
constants
'
][
cname
][
"
file-path
"
]
=
f
"
{
slopes_ff_from_files
}
/slopesff_bpmask_module_
{
qm
}
.h5
"
mdata_dict
[
'
constants
'
][
cname
][
"
creation-time
"
]
=
"
00:00:00
"
mdata_dict
[
'
constants
'
][
cname
][
"
creation-time
"
]
=
"
00:00:00
"
else
:
else
:
condition
=
getattr
(
Conditions
,
cval
[
2
][
0
]).
AGIPD
(
**
cval
[
2
][
1
])
condition
=
getattr
(
Conditions
,
cval
[
2
][
0
]).
AGIPD
(
**
cval
[
2
][
1
])
co
,
mdata
=
tools
.
get_from_db
(
co
,
mdata
=
tools
.
get_from_db
(
karabo_id
,
karabo_id
,
karabo_da
,
karabo_da
,
getattr
(
Constants
.
AGIPD
,
cname
)(),
getattr
(
Constants
.
AGIPD
,
cname
)(),
condition
,
condition
,
getattr
(
np
,
cval
[
0
])(
cval
[
1
]),
getattr
(
np
,
cval
[
0
])(
cval
[
1
]),
cal_db_interface
,
cal_db_interface
,
creation_time
,
creation_time
,
meta_only
=
True
,
meta_only
=
True
,
verbosity
=
1
,
verbosity
=
1
,
)
)
mdata_const
=
mdata
.
calibration_constant_version
mdata_const
=
mdata
.
calibration_constant_version
# check if constant was sucessfully retrieved.
# check if constant was sucessfully retrieved.
if
mdata
.
comm_db_success
:
if
mdata
.
comm_db_success
:
mdata_dict
[
'
constants
'
][
cname
][
"
file-path
"
]
=
f
"
{
mdata_const
.
hdf5path
}
"
\
mdata_dict
[
'
constants
'
][
cname
][
"
file-path
"
]
=
f
"
{
mdata_const
.
hdf5path
}
"
\
f
"
{
mdata_const
.
filename
}
"
f
"
{
mdata_const
.
filename
}
"
mdata_dict
[
'
constants
'
][
cname
][
"
creation-time
"
]
=
f
"
{
mdata_const
.
begin_at
}
"
mdata_dict
[
'
constants
'
][
cname
][
"
creation-time
"
]
=
f
"
{
mdata_const
.
begin_at
}
"
mdata_dict
[
'
physical-detector-unit
'
]
=
mdata_const
.
device_name
mdata_dict
[
'
physical-detector-unit
'
]
=
mdata_const
.
device_name
else
:
else
:
mdata_dict
[
'
constants
'
][
cname
][
"
file-path
"
]
=
const_dict
[
cname
][:
2
]
mdata_dict
[
'
constants
'
][
cname
][
"
file-path
"
]
=
const_dict
[
cname
][:
2
]
mdata_dict
[
'
constants
'
][
cname
][
"
creation-time
"
]
=
None
mdata_dict
[
'
constants
'
][
cname
][
"
creation-time
"
]
=
None
return
qm
,
mdata_dict
,
karabo_da
,
acq_rate
,
local_max_cells
,
err
return
qm
,
mdata_dict
,
karabo_da
,
acq_rate
,
local_max_cells
,
err
```
```
%% Cell type:code id: tags:
%% Cell type:code id: tags:
```
python
```
python
# set everything up filewise
# set everything up filewise
mapped_files
,
_
,
_
,
_
,
_
=
tools
.
map_modules_from_folder
(
mapped_files
,
_
,
_
,
_
,
_
=
tools
.
map_modules_from_folder
(
str
(
in_folder
),
run
,
path_template
,
karabo_da
,
sequences
str
(
in_folder
),
run
,
path_template
,
karabo_da
,
sequences
)
)
pc_bools
=
[
corr_bools
.
get
(
"
rel_gain
"
),
pc_bools
=
[
corr_bools
.
get
(
"
rel_gain
"
),
corr_bools
.
get
(
"
adjust_mg_baseline
"
),
corr_bools
.
get
(
"
adjust_mg_baseline
"
),
corr_bools
.
get
(
'
blc_noise
'
),
corr_bools
.
get
(
'
blc_noise
'
),
corr_bools
.
get
(
'
blc_hmatch
'
),
corr_bools
.
get
(
'
blc_hmatch
'
),
corr_bools
.
get
(
'
blc_stripes
'
),
corr_bools
.
get
(
'
blc_stripes
'
),
melt_snow
]
melt_snow
]
inp
=
[]
inp
=
[]
only_dark
=
False
only_dark
=
False
nodb_with_dark
=
False
nodb_with_dark
=
False
if
not
nodb
:
if
not
nodb
:
only_dark
=
(
calfile
!=
""
)
only_dark
=
(
calfile
!=
""
)
if
calfile
!=
""
and
not
corr_bools
[
"
only_offset
"
]:
if
calfile
!=
""
and
not
corr_bools
[
"
only_offset
"
]:
nodb_with_dark
=
nodb
nodb_with_dark
=
nodb
# A dict to connect virtual device
# A dict to connect virtual device
# to actual device name.
# to actual device name.
for
module_index
,
k_da
in
zip
(
modules
,
karabo_da
):
for
module_index
,
k_da
in
zip
(
modules
,
karabo_da
):
qm
=
tools
.
module_index_to_qm
(
module_index
)
qm
=
tools
.
module_index_to_qm
(
module_index
)
if
qm
in
mapped_files
and
not
mapped_files
[
qm
].
empty
():
if
qm
in
mapped_files
and
not
mapped_files
[
qm
].
empty
():
device
=
getattr
(
getattr
(
Detectors
,
dinstance
),
qm
)
device
=
getattr
(
getattr
(
Detectors
,
dinstance
),
qm
)
# TODO: make map_modules_from_folder just return list(s)
# TODO: make map_modules_from_folder just return list(s)
qm_files
=
[
Path
(
mapped_files
[
qm
].
get
())
for
_
in
range
(
mapped_files
[
qm
].
qsize
())]
qm_files
=
[
Path
(
mapped_files
[
qm
].
get
())
for
_
in
range
(
mapped_files
[
qm
].
qsize
())]
else
:
else
:
print
(
f
"
Skipping
{
qm
}
"
)
print
(
f
"
Skipping
{
qm
}
"
)
continue
continue
inp
.
append
((
qm_files
,
qm
,
k_da
,
module_index
))
inp
.
append
((
qm_files
,
qm
,
k_da
,
module_index
))
```
```
%% Cell type:code id: tags:
%% Cell type:code id: tags:
```
python
```
python
with
multiprocessing
.
Pool
(
processes
=
nmods
)
as
pool
:
with
multiprocessing
.
Pool
(
processes
=
nmods
)
as
pool
:
results
=
pool
.
starmap
(
retrieve_constants
,
inp
)
results
=
pool
.
starmap
(
retrieve_constants
,
inp
)
```
```
%% Cell type:code id: tags:
%% Cell type:code id: tags:
```
python
```
python
mod_dev
=
dict
()
mod_dev
=
dict
()
mdata_dict
=
dict
()
mdata_dict
=
dict
()
for
r
in
results
:
for
qm
,
md_dict
,
karabo_da
,
acq_rate
,
max_cells
,
err
in
results
:
if
r
:
mod_dev
[
karabo_da
]
=
{
"
mod
"
:
qm
,
"
err
"
:
err
}
qm
,
md_dict
,
karabo_da
,
acq_rate
,
max_cells
,
err
=
r
if
err
:
mod_dev
[
karabo_da
]
=
{
"
mod
"
:
qm
,
"
err
"
:
err
}
print
(
f
"
Error for module
{
qm
}
:
{
err
}
"
)
if
err
:
mdata_dict
[
karabo_da
]
=
md_dict
print
(
f
"
Error for module
{
qm
}
:
{
err
}
"
)
mdata_dict
[
karabo_da
]
=
md_dict
# check if it is requested not to retrieve any constants from the database
# check if it is requested not to retrieve any constants from the database
if
nodb_with_dark
:
if
nodb_with_dark
:
print
(
"
No constants were retrieved as calibrated files will be used.
"
)
print
(
"
No constants were retrieved as calibrated files will be used.
"
)
else
:
else
:
metadata
.
update
({
"
retrieved-constants
"
:
mdata_dict
})
metadata
.
update
({
"
retrieved-constants
"
:
mdata_dict
})
print
(
"
\n
Retrieved constants for modules:
"
,
print
(
"
\n
Retrieved constants for modules:
"
,
'
,
'
.
join
([
tools
.
module_index_to_qm
(
x
)
for
x
in
modules
]))
'
,
'
.
join
([
tools
.
module_index_to_qm
(
x
)
for
x
in
modules
]))
print
(
f
"
Operating conditions are:
"
)
print
(
f
"
Operating conditions are:
"
)
print
(
f
"
• Bias voltage:
{
bias_voltage
}
"
)
print
(
f
"
• Bias voltage:
{
bias_voltage
}
"
)
print
(
f
"
• Memory cells:
{
max_cells
}
"
)
print
(
f
"
• Memory cells:
{
max_cells
}
"
)
print
(
f
"
• Acquisition rate:
{
acq_rate
}
"
)
print
(
f
"
• Acquisition rate:
{
acq_rate
}
"
)
print
(
f
"
• Gain mode:
{
gain_mode
.
name
}
"
)
print
(
f
"
• Gain mode:
{
gain_mode
.
name
}
"
)
print
(
f
"
• Gain setting:
{
gain_setting
}
"
)
print
(
f
"
• Gain setting:
{
gain_setting
}
"
)
print
(
f
"
• Photon Energy:
{
photon_energy
}
"
)
print
(
f
"
• Photon Energy:
{
photon_energy
}
"
)
print
(
"
Constant metadata is saved under
\"
retrieved-constants
\"
in calibration_metadata.yml
\n
"
)
print
(
"
Constant metadata is saved under
\"
retrieved-constants
\"
in calibration_metadata.yml
\n
"
)
```
```
%% Cell type:code id: tags:
%% Cell type:code id: tags:
```
python
```
python
print
(
"
Constants are retrieved with creation time:
"
)
print
(
"
Constants are retrieved with creation time:
"
)
i
=
0
timestamps
=
{}
timestamps
=
{}
for
k_da
,
dinfo
in
mod_dev
.
items
():
for
k_da
,
dinfo
in
mod_dev
.
items
():
print
(
dinfo
[
"
mod
"
],
"
:
"
)
print
(
f
"
{
dinfo
[
'
mod
'
]
}
:
"
)
module_timestamps
=
{}
module_timestamps
=
{}
module_name
=
dinfo
[
"
mod
"
]
module_name
=
dinfo
[
"
mod
"
]
if
k_da
in
mdata_dict
:
if
k_da
in
mdata_dict
:
for
cname
,
mdata
in
mdata_dict
[
k_da
][
"
constants
"
].
items
():
for
cname
,
mdata
in
mdata_dict
[
k_da
][
"
constants
"
].
items
():
if
hasattr
(
mdata
[
"
creation-time
"
],
'
strftime
'
):
if
hasattr
(
mdata
[
"
creation-time
"
],
'
strftime
'
):
mdata
[
"
creation-time
"
]
=
mdata
[
"
creation-time
"
].
strftime
(
'
%y-%m-%d %H:%M
'
)
mdata
[
"
creation-time
"
]
=
mdata
[
"
creation-time
"
].
strftime
(
'
%y-%m-%d %H:%M
'
)
print
(
f
'
{
cname
:
.
<
12
s
}
'
,
mdata
[
"
creation-time
"
])
print
(
f
'
{
cname
:
.
<
12
s
}
'
,
mdata
[
"
creation-time
"
])
# Store few time stamps if exists
# Add NA to keep array structure
for
cname
in
[
'
Offset
'
,
'
SlopesPC
'
,
'
SlopesFF
'
]:
for
cname
in
[
'
Offset
'
,
'
SlopesPC
'
,
'
SlopesFF
'
]:
if
k_da
not
in
mdata_dict
or
dinfo
[
"
err
"
]:
if
k_da
not
in
mdata_dict
or
dinfo
[
"
err
"
]:
module_timestamps
[
cname
]
=
"
Err
"
module_timestamps
[
cname
]
=
"
Err
"
else
:
else
:
if
cname
in
mdata_dict
[
k_da
]:
if
cname
in
mdata_dict
[
k_da
]:
if
mdata_dict
[
k_da
][
cname
][
"
creation-time
"
]:
if
mdata_dict
[
k_da
][
cname
][
"
creation-time
"
]:
module_timestamps
[
cname
]
=
mdata_dict
[
k_da
][
cname
][
"
creation-time
"
]
module_timestamps
[
cname
]
=
mdata_dict
[
k_da
][
cname
][
"
creation-time
"
]
else
:
else
:
module_timestamps
[
cname
]
=
"
NA
"
module_timestamps
[
cname
]
=
"
NA
"
else
:
else
:
module_timestamps
[
cname
]
=
"
NA
"
module_timestamps
[
cname
]
=
"
NA
"
timestamps
[
module_name
]
=
module_timestamps
timestamps
[
module_name
]
=
module_timestamps
i
+=
1
if
sequences
:
seq_num
=
sequences
[
0
]
else
:
# if sequences[0] changed to None as it was -1
seq_num
=
0
time_summary
=
metadata
.
setdefault
(
"
retrieved-constants
"
,
{}).
setdefault
(
"
time-summary
"
,
{})
time_summary
=
metadata
.
setdefault
(
"
retrieved-constants
"
,
{}).
setdefault
(
"
time-summary
"
,
{})
time_summary
[
"
SAll
"
]
=
timestamps
time_summary
[
"
SAll
"
]
=
timestamps
metadata
.
save
()
metadata
.
save
()
```
```
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment