From f86be23d1567fb0b36d96c84a6118ee884ee40ee Mon Sep 17 00:00:00 2001
From: Thomas Kluyver <thomas@kluyver.me.uk>
Date: Fri, 16 Dec 2022 17:47:43 +0100
Subject: [PATCH] Add support for saving metadata fragments & merging into
calibration_metadata.yml
---
src/cal_tools/tools.py | 31 +++++++++++++++++++++++++++++++
1 file changed, 31 insertions(+)
diff --git a/src/cal_tools/tools.py b/src/cal_tools/tools.py
index 2d15866ac..0760d48a7 100644
--- a/src/cal_tools/tools.py
+++ b/src/cal_tools/tools.py
@@ -10,6 +10,7 @@ from os import environ, listdir, path
from os.path import isfile
from pathlib import Path
from queue import Queue
+from tempfile import NamedTemporaryFile
from time import sleep
from typing import List, Optional, Tuple, Union
from urllib.parse import urljoin
@@ -811,6 +812,14 @@ def module_index_to_qm(index: int, total_modules: int = 16):
return f"Q{quad+1}M{mod+1}"
+def recursive_update(target: dict, source: dict):
+ for k, v2 in source.items():
+ v1 = target.get(k, None)
+ if isinstance(v1, dict) and isinstance(v2, dict):
+ recursive_update(v1, v2)
+ else:
+ target[k] = v2
+
class CalibrationMetadata(dict):
"""Convenience class: dictionary stored in metadata YAML file
@@ -847,6 +856,28 @@ class CalibrationMetadata(dict):
with (copy_dir / self._yaml_fn.name).open("w") as fd:
yaml.safe_dump(dict(self), fd)
+ def add_fragment(self, data: dict):
+ """Save metadata to a separate 'fragment' file to be merged later
+
+ Avoids a risk of corrupting the main file by writing in parallel.
+ """
+ with NamedTemporaryFile("w", dir=self._yaml_fn.parent,
+ prefix='metadata_frag_', suffix='.yml', delete=False) as fd:
+ yaml.safe_dump(data, fd)
+
+ def gather_fragments(self):
+ """Merge in fragments saved by add_fragment(), then delete them"""
+ frag_files = list(self._yaml_fn.parent.glob('metadata_frag_*.yml'))
+ for fn in frag_files:
+ with fn.open("r") as fd:
+ data = yaml.safe_load(fd)
+ recursive_update(self, data)
+
+ self.save()
+
+ for fn in frag_files:
+ fn.unlink()
+
def save_constant_metadata(
retrieved_constants: dict,
--
GitLab