Wait for all assemblers to be ACTIVE before sending geometry

We've recently observed a race condition between the assemblers coming up and connected to the geometry device, and the geometry device sending its geometry. This MR is an attempt at avoiding this race condition by waiting (up to 3s) for the assemblers to be active.

@hammerd

src/calng/CalibrationManager.py

@@ -5,6 +5,7 @@
 #############################################################################
 
 from asyncio import gather, wait_for, TimeoutError as AsyncTimeoutError
+from contextlib import ExitStack
 from collections import defaultdict
 from collections.abc import Hashable
 from datetime import datetime
@@ -25,8 +26,9 @@ from karabo.middlelayer import (
     AccessMode, AccessLevel, Assignment, DaqPolicy, State, Unit,
     UInt16, UInt32, Bool, Float, Double, String, VectorString, VectorHash,
     background, call, callNoWait, setNoWait, sleep, instantiate, slot, coslot,
-    get_property, getTopology, getConfiguration, getConfigurationFromPast,
-    getConfigurationFromName, getInstanceInfo)
+    get_property, getDevice, getTopology, getConfiguration,
+    getConfigurationFromPast, getConfigurationFromName, getInstanceInfo,
+    waitUntil)
 
 from . import scenes
 from ._version import version as deviceVersion
@@ -1056,6 +1058,22 @@ class CalibrationManager(DeviceClientBase, Device):
             if info['status'].startswith(state):
                 return
 
+    async def _ensure_assemblers_active(self):
+        """Ensure assemblers are in ACTIVE state."""
+
+        # This coroutine may throw an AsyncTimeoutError to be handled
+        # by the calling site.
+
+        with ExitStack() as stack:
+            assemblers = [stack.enter_context(
+                await wait_for(getDevice(device_id), timeout=3))
+                for device_id in self._assembler_device_ids]
+
+            await wait_for(gather(*[
+                waitUntil(lambda: assembler.state == State.ACTIVE)
+                for assembler in assemblers]),
+                timeout=3)
+
     async def _check_servers(self):
         """Validate device server configuration."""
 
@@ -1390,6 +1408,14 @@ class CalibrationManager(DeviceClientBase, Device):
         # Force managed DAQ settings.
         await self._apply_managed_values(daq=True)
 
+        # Try waiting until assemblers are active.
+        try:
+            await self._ensure_assemblers_active()
+        except AsyncTimeoutError:
+            self._set_status('One or more assemblers missing or not in '
+                             'active state after timeout - geometry may '
+                             'require manual update')
+
         # Ask the geometry device to re-send its geometry.
         callNoWait(self.geometryDevice.value, 'sendGeometry')