Add support for saving metadata fragments & merging into calibration_metadata.yml

src/cal_tools/tools.py

@@ -10,6 +10,7 @@ from os import environ, listdir, path
 from os.path import isfile
 from pathlib import Path
 from queue import Queue
+from tempfile import NamedTemporaryFile
 from time import sleep
 from typing import List, Optional, Tuple, Union
 from urllib.parse import urljoin
@@ -811,6 +812,14 @@ def module_index_to_qm(index: int, total_modules: int = 16):
     return f"Q{quad+1}M{mod+1}"
 
 
+def recursive_update(target: dict, source: dict):
+    for k, v2 in source.items():
+        v1 = target.get(k, None)
+        if isinstance(v1, dict) and isinstance(v2, dict):
+            recursive_update(v1, v2)
+        else:
+            target[k] = v2
+
 class CalibrationMetadata(dict):
     """Convenience class: dictionary stored in metadata YAML file
 
@@ -847,6 +856,28 @@ class CalibrationMetadata(dict):
         with (copy_dir / self._yaml_fn.name).open("w") as fd:
             yaml.safe_dump(dict(self), fd)
 
+    def add_fragment(self, data: dict):
+        """Save metadata to a separate 'fragment' file to be merged later
+
+        Avoids a risk of corrupting the main file by writing in parallel.
+        """
+        with NamedTemporaryFile("w", dir=self._yaml_fn.parent,
+                    prefix='metadata_frag_', suffix='.yml', delete=False) as fd:
+            yaml.safe_dump(data, fd)
+
+    def gather_fragments(self):
+        """Merge in fragments saved by add_fragment(), then delete them"""
+        frag_files = list(self._yaml_fn.parent.glob('metadata_frag_*.yml'))
+        for fn in frag_files:
+            with fn.open("r") as fd:
+                data = yaml.safe_load(fd)
+                recursive_update(self, data)
+
+        self.save()
+
+        for fn in frag_files:
+            fn.unlink()
+
 
 def save_constant_metadata(
     retrieved_constants: dict,